{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "dfccd8e6",
   "metadata": {},
   "source": [
    "# Fine-Tuning GPT-2 on Encrypted Data with LoRA and Concrete ML\n",
    "\n",
    "In this notebook, we perform fine-tuning of a GPT-2 model using LoRA and Concrete ML."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "eca73e44",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x306f29190>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Import necessary libraries\n",
    "import math\n",
    "import shutil\n",
    "from pathlib import Path\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "from datasets import Dataset\n",
    "from peft import LoraConfig, get_peft_model\n",
    "from tqdm import tqdm\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer, Trainer, TrainingArguments\n",
    "from utils_lora import generate_and_print, print_weights_and_size\n",
    "\n",
    "from concrete.ml.torch.hybrid_model import HybridFHEModel\n",
    "from concrete.ml.torch.lora import LoraTraining, get_remote_names\n",
    "\n",
    "# Set random seed for reproducibility\n",
    "SEED = 0\n",
    "torch.manual_seed(SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8b965a1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load pre-trained GPT-2 model and tokenizer\n",
    "model_name = \"gpt2\"\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
    "model = AutoModelForCausalLM.from_pretrained(model_name)\n",
    "\n",
    "# Ensure tokenizer has a pad token\n",
    "if tokenizer.pad_token is None:\n",
    "    tokenizer.pad_token = tokenizer.eos_token\n",
    "model.config.pad_token_id = model.config.eos_token_id\n",
    "\n",
    "# Freeze model weights\n",
    "for param in model.parameters():\n",
    "    param.requires_grad = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2337a6b4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prompt: What is FHE?\n",
      "Response: FHE is a type of \"lightweight\" powerplant that is designed to be used in a variety of applications. It is a hybrid\n",
      "\n"
     ]
    }
   ],
   "source": [
    "_ = generate_and_print(prompt=\"What is FHE?\", model=model, tokenizer=tokenizer, seed=SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "20564b59",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Apply LoRA to the model\n",
    "# target_modules can be set to \"all-linear\"\n",
    "# to target all modules. By default only the\n",
    "# c_attn projection are fine-tuned with lora.\n",
    "peft_config = LoraConfig(\n",
    "    r=8,\n",
    "    lora_alpha=32,\n",
    "    lora_dropout=0.1,\n",
    "    bias=\"none\",\n",
    "    task_type=\"CAUSAL_LM\",\n",
    ")\n",
    "peft_model = get_peft_model(model, peft_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5ac49f9d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LoRA layers detected in the model.\n"
     ]
    }
   ],
   "source": [
    "# Set up LoRA training\n",
    "lora_training = LoraTraining(peft_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d10d71e8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2fe009bd19a14ddcb42ef4646dbad9d4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/34 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Prepare dataset for fine-tuning\n",
    "BLOCK_SIZE = 128\n",
    "\n",
    "# Read lines from the file\n",
    "with open(\"data_finetune/what_is_fhe.txt\", \"r\", encoding=\"utf-8\") as f:\n",
    "    lines = f.readlines()\n",
    "\n",
    "# Remove empty lines and strip whitespace\n",
    "lines = [line.strip() for line in lines if line.strip()]\n",
    "\n",
    "# Group lines into question-answer pairs\n",
    "examples = []\n",
    "for i in range(0, len(lines) - 1, 2):\n",
    "    question = lines[i]\n",
    "    answer = lines[i + 1]\n",
    "    examples.append({\"question\": question, \"answer\": answer})\n",
    "\n",
    "# Create a Dataset object from the list of examples\n",
    "dataset = Dataset.from_list(examples)\n",
    "\n",
    "\n",
    "# Tokenization function\n",
    "def tokenize_function(examples):\n",
    "    input_ids_list = []\n",
    "    labels_list = []\n",
    "    attention_masks_list = []\n",
    "    for question, answer in zip(examples[\"question\"], examples[\"answer\"]):\n",
    "        # Tokenize question and answer separately\n",
    "        question_encoding = tokenizer(\n",
    "            question, add_special_tokens=False, truncation=True, max_length=BLOCK_SIZE // 2\n",
    "        )\n",
    "        answer_encoding = tokenizer(\n",
    "            answer, add_special_tokens=False, truncation=True, max_length=BLOCK_SIZE // 2 - 1\n",
    "        )\n",
    "\n",
    "        # Build input_ids\n",
    "        input_ids = (\n",
    "            question_encoding[\"input_ids\"]\n",
    "            + [tokenizer.eos_token_id]\n",
    "            + answer_encoding[\"input_ids\"]\n",
    "            + [tokenizer.eos_token_id]\n",
    "        )\n",
    "\n",
    "        # Build labels: -100 for question tokens and eos token after question\n",
    "        labels = (\n",
    "            [-100] * len(question_encoding[\"input_ids\"])\n",
    "            + [-100]  # For the eos token after question\n",
    "            + answer_encoding[\"input_ids\"]\n",
    "            + [tokenizer.eos_token_id]\n",
    "        )\n",
    "\n",
    "        # Create attention mask: 1 for real tokens, 0 for padding\n",
    "        attention_mask = [1] * len(input_ids)\n",
    "\n",
    "        # Pad/truncate to BLOCK_SIZE\n",
    "        padding_length = BLOCK_SIZE - len(input_ids)\n",
    "        if padding_length > 0:\n",
    "            input_ids += [tokenizer.pad_token_id] * padding_length\n",
    "            labels += [-100] * padding_length\n",
    "            attention_mask += [0] * padding_length\n",
    "        else:\n",
    "            input_ids = input_ids[:BLOCK_SIZE]\n",
    "            labels = labels[:BLOCK_SIZE]\n",
    "            attention_mask = attention_mask[:BLOCK_SIZE]\n",
    "\n",
    "        input_ids_list.append(input_ids)\n",
    "        labels_list.append(labels)\n",
    "        attention_masks_list.append(attention_mask)\n",
    "\n",
    "    return {\n",
    "        \"input_ids\": input_ids_list,\n",
    "        \"labels\": labels_list,\n",
    "        \"attention_mask\": attention_masks_list,\n",
    "    }\n",
    "\n",
    "\n",
    "# Apply the tokenization\n",
    "tokenized_datasets = dataset.map(\n",
    "    tokenize_function, batched=True, remove_columns=[\"question\", \"answer\"]\n",
    ")\n",
    "\n",
    "# Since we've already handled padding and labels, we can use a custom data collator\n",
    "\n",
    "\n",
    "def data_collator(features):\n",
    "    batch = {}\n",
    "    batch[\"input_ids\"] = torch.tensor([f[\"input_ids\"] for f in features], dtype=torch.long)\n",
    "    batch[\"labels\"] = torch.tensor([f[\"labels\"] for f in features], dtype=torch.long)\n",
    "    batch[\"attention_mask\"] = torch.tensor(\n",
    "        [f[\"attention_mask\"] for f in features], dtype=torch.long\n",
    "    )\n",
    "    return batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8a01acd1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define training arguments\n",
    "EPOCHS = 20\n",
    "PER_DEVICE_TRAIN_BATCH_SIZE = 4\n",
    "\n",
    "training_args = TrainingArguments(\n",
    "    output_dir=\"./checkpoints\",\n",
    "    num_train_epochs=EPOCHS,\n",
    "    per_device_train_batch_size=PER_DEVICE_TRAIN_BATCH_SIZE,\n",
    "    gradient_accumulation_steps=1,\n",
    "    save_total_limit=1,\n",
    "    use_cpu=True,\n",
    "    learning_rate=2e-3,\n",
    "    lr_scheduler_type=\"linear\",\n",
    "    seed=SEED,\n",
    "    data_seed=SEED,\n",
    "    warmup_steps=10,\n",
    "    weight_decay=0.01,\n",
    "    prediction_loss_only=True,\n",
    ")\n",
    "\n",
    "\n",
    "def causal_lm_loss(logits, labels, ignore_index=-100):\n",
    "    # Shift logits and labels for next-token prediction\n",
    "    shift_logits = logits[..., :-1, :].contiguous()\n",
    "    shift_labels = labels[..., 1:].contiguous()\n",
    "\n",
    "    # Flatten the tensors\n",
    "    shift_logits = shift_logits.view(-1, shift_logits.size(-1))\n",
    "    shift_labels = shift_labels.view(-1)\n",
    "\n",
    "    # Compute the loss, ignoring padding tokens\n",
    "    loss = torch.nn.functional.cross_entropy(\n",
    "        shift_logits, shift_labels, ignore_index=ignore_index, reduction=\"mean\"\n",
    "    )\n",
    "\n",
    "    return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3c8864b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize Trainer\n",
    "trainer = Trainer(\n",
    "    model=peft_model,\n",
    "    args=training_args,\n",
    "    train_dataset=tokenized_datasets,\n",
    "    data_collator=data_collator,\n",
    ")\n",
    "\n",
    "# Prepare for training\n",
    "train_dataloader = trainer.get_train_dataloader()\n",
    "\n",
    "len_dataloader = len(train_dataloader)\n",
    "num_update_steps_per_epoch = len_dataloader // training_args.gradient_accumulation_steps\n",
    "num_update_steps_per_epoch = max(num_update_steps_per_epoch, 1)\n",
    "max_steps = math.ceil(training_args.num_train_epochs * num_update_steps_per_epoch)\n",
    "\n",
    "trainer.create_optimizer_and_scheduler(num_training_steps=max_steps)\n",
    "\n",
    "lr_scheduler = trainer.lr_scheduler\n",
    "optimizer = trainer.optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ae2094a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the names of the remote modules (layers to be converted to FHE)\n",
    "remote_names = get_remote_names(lora_training, include_embedding_layers=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a21298ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the HybridFHEModel with the specified remote modules\n",
    "hybrid_model = HybridFHEModel(lora_training, module_names=remote_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "56ec41b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prepare input data for calibration\n",
    "input_tensor = torch.randint(\n",
    "    0, tokenizer.vocab_size, (PER_DEVICE_TRAIN_BATCH_SIZE, BLOCK_SIZE), dtype=torch.long\n",
    ")\n",
    "label_tensor = torch.randint(\n",
    "    0, tokenizer.vocab_size, (PER_DEVICE_TRAIN_BATCH_SIZE, BLOCK_SIZE), dtype=torch.long\n",
    ")\n",
    "attention_mask = torch.ones((PER_DEVICE_TRAIN_BATCH_SIZE, BLOCK_SIZE), dtype=torch.long)\n",
    "\n",
    "inputset = {\"input_ids\": input_tensor, \"attention_mask\": attention_mask, \"labels\": label_tensor}\n",
    "\n",
    "# inputset = (input_tensor, label_tensor, attention_mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "20dfe2d8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "464a3a0b38674439bee9c3ccd475b335",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Compiling FHE layers:   0%|          | 0/95 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Calibrate and compile the model\n",
    "hybrid_model.compile_model(inputset, n_bits=8, use_dynamic_quantization=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "18e450e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_custom_model(\n",
    "    hybrid_model, train_dataloader, training_args, tokenizer, fhe=\"disable\"\n",
    "):  # pylint: disable=too-many-locals\n",
    "    device = \"cpu\"\n",
    "    hybrid_model.model.to(device)\n",
    "\n",
    "    # Training loop\n",
    "    peft_model.train()\n",
    "    total_epochs = int(training_args.num_train_epochs)\n",
    "    epoch_pbar = tqdm(total=total_epochs, desc=\"Training Progress\", position=0)\n",
    "\n",
    "    total_batched_samples = 0\n",
    "    epoch_losses = []\n",
    "\n",
    "    # Generate text before the first epoch\n",
    "    print(\"Generating text before the first epoch:\\n\")\n",
    "    prompt = \"What is FHE?\"\n",
    "    hybrid_model.set_fhe_mode(\"disable\")\n",
    "    generate_and_print(prompt, peft_model, tokenizer, SEED)\n",
    "    hybrid_model.set_fhe_mode(fhe)\n",
    "\n",
    "    for epoch in range(total_epochs):\n",
    "        total_loss = 0\n",
    "        grad_norms = []\n",
    "\n",
    "        for _, batch in enumerate(train_dataloader):\n",
    "            total_batched_samples += 1\n",
    "            batch = {k: v.to(device) for k, v in batch.items()}\n",
    "\n",
    "            # Zero the gradients\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "            # Forward pass\n",
    "            loss, grad_norm = hybrid_model(batch, fhe=fhe)\n",
    "\n",
    "            # Optimizer step\n",
    "            optimizer.step()\n",
    "\n",
    "            # Learning rate scheduler step\n",
    "            lr_scheduler.step()\n",
    "\n",
    "            total_loss += loss.item()\n",
    "            if grad_norm is not None:\n",
    "                grad_norms.append(grad_norm)\n",
    "\n",
    "        # Get current learning rate\n",
    "        current_lr = lr_scheduler.get_last_lr()[0]\n",
    "\n",
    "        # Get last grad norm\n",
    "        current_grad_norm = grad_norms[-1] if grad_norms else None\n",
    "\n",
    "        # Store the total loss for this epoch\n",
    "        epoch_losses.append(total_loss)\n",
    "\n",
    "        # Log epoch results\n",
    "        print(\n",
    "            f\"Epoch {epoch + 1}/{training_args.num_train_epochs}, \"\n",
    "            f\"Loss: {total_loss:.4f}, grad norm: {current_grad_norm}, lr: {current_lr}\"\n",
    "        )\n",
    "\n",
    "        # Generate text after each epoch\n",
    "        prompt = \"What is FHE?\"\n",
    "        hybrid_model.set_fhe_mode(\"disable\")\n",
    "        generate_and_print(prompt, peft_model, tokenizer, SEED)\n",
    "        hybrid_model.set_fhe_mode(fhe)\n",
    "\n",
    "        print(\"\\n\" + \"-\" * 50)  # Separator for readability\n",
    "        epoch_pbar.update(1)\n",
    "\n",
    "    # Save model checkpoint\n",
    "    if training_args.output_dir is not None:\n",
    "        save_path = f\"{training_args.output_dir}/checkpoint-{epoch + 1}\"\n",
    "        peft_model.save_pretrained(save_path)\n",
    "\n",
    "    epoch_pbar.close()\n",
    "\n",
    "    # Plot the loss evolution\n",
    "    plt.figure(figsize=(10, 6))\n",
    "    plt.plot(range(1, total_epochs + 1), epoch_losses, marker=\"o\")\n",
    "    plt.title(\"Loss Evolution During Training\")\n",
    "    plt.xlabel(\"Epoch\")\n",
    "    plt.ylabel(\"Total Loss\")\n",
    "    plt.grid(True)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "0ca82a81",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Progress:   0%|          | 0/20 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating text before the first epoch:\n",
      "\n",
      "Prompt: What is FHE?\n",
      "Response: FHE is a \"\n",
      "FHE is a \"\n",
      "\n",
      "\n",
      "FHE is a \"\n",
      "\n",
      "FHE is a\n",
      "\n",
      "F\n",
      "\n",
      "Epoch 1/20, Loss: 31.9151, grad norm: None, lr: 0.0018000000000000002\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Progress:   5%|▌         | 1/20 [00:18<05:45, 18.16s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prompt: What is FHE?\n",
      "Response: FHE is a type of type of data storage that is used for storing data about information about a variety of information, including the information about\n",
      "\n",
      "\n",
      "--------------------------------------------------\n",
      "Epoch 2/20, Loss: 26.1764, grad norm: None, lr: 0.0019058823529411763\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Progress:  10%|█         | 2/20 [00:28<04:02, 13.47s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prompt: What is FHE?\n",
      "Response: FHE is a self-aware self-aware self-aware self-aware self-aware self-aware self-aware self-aware\n",
      "\n",
      "\n",
      "--------------------------------------------------\n",
      "Epoch 3/20, Loss: 22.6590, grad norm: None, lr: 0.0018000000000000002\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Progress:  15%|█▌        | 3/20 [00:38<03:22, 11.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prompt: What is FHE?\n",
      "Response: FHE is a mathematical model that enables mathematical modeling of data structures. It enables modeling of computations in ways that are computable on the computer,\n",
      "\n",
      "\n",
      "--------------------------------------------------\n",
      "Epoch 4/20, Loss: 20.1917, grad norm: None, lr: 0.0016941176470588236\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Progress:  20%|██        | 4/20 [00:48<03:00, 11.30s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prompt: What is FHE?\n",
      "Response: FHE is a type of encryption scheme that allows the government to decrypt and decrypted data without revealing its contents, even if the data is\n",
      "\n",
      "\n",
      "--------------------------------------------------\n",
      "Epoch 5/20, Loss: 19.0490, grad norm: None, lr: 0.001588235294117647\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Progress:  25%|██▌       | 5/20 [00:59<02:44, 10.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prompt: What is FHE?\n",
      "Response: Fully Homomorphic Encryption (FHE) is a new concept in computing that aims to improve the speed of encryption by reducing the processing time required for\n",
      "\n",
      "\n",
      "--------------------------------------------------\n",
      "Epoch 6/20, Loss: 16.8715, grad norm: None, lr: 0.0014823529411764707\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Progress:  30%|███       | 6/20 [01:09<02:32, 10.87s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prompt: What is FHE?\n",
      "Response: Fully Homomorphic Encryption (FHE) is a new approach for securely storing encrypted data in secure environments. FHE uses a mathematical model of encrypted\n",
      "\n",
      "\n",
      "--------------------------------------------------\n",
      "Epoch 7/20, Loss: 15.1334, grad norm: None, lr: 0.001376470588235294\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Progress:  35%|███▌      | 7/20 [01:20<02:21, 10.88s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prompt: What is FHE?\n",
      "Response: Fully Homomorphic Encryption (FHE) is a new approach that involves computing the operation of encryption keys. This makes it possible to process encrypted data\n",
      "\n",
      "\n",
      "--------------------------------------------------\n",
      "Epoch 8/20, Loss: 14.4121, grad norm: None, lr: 0.0012705882352941175\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Progress:  40%|████      | 8/20 [01:31<02:10, 10.84s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prompt: What is FHE?\n",
      "Response: Fully Homomorphic Encryption (FHE) is a fast-growing fast-growing fast-growing fast-growing fast-growing fast-growing fast\n",
      "\n",
      "\n",
      "--------------------------------------------------\n",
      "Epoch 9/20, Loss: 13.2743, grad norm: None, lr: 0.0011647058823529412\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Progress:  45%|████▌     | 9/20 [01:42<01:59, 10.90s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prompt: What is FHE?\n",
      "Response: Fully Homomorphic Encryption (FHE) is a new approach to computational training by enabling computations on encrypted data. This gives a unique insight into\n",
      "\n",
      "\n",
      "--------------------------------------------------\n",
      "Epoch 10/20, Loss: 12.2570, grad norm: None, lr: 0.0010588235294117648\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Progress:  50%|█████     | 10/20 [01:53<01:49, 10.91s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prompt: What is FHE?\n",
      "Response: FHE is a type of encryption technique that allows users to encrypt their data without ever decrypting it. This type of encryption is also known as dec\n",
      "\n",
      "\n",
      "--------------------------------------------------\n",
      "Epoch 11/20, Loss: 11.0030, grad norm: None, lr: 0.0009529411764705882\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Progress:  55%|█████▌    | 11/20 [02:04<01:39, 11.09s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prompt: What is FHE?\n",
      "Response: Fully Homomorphic Encryption (FHE) provides a high level of privacy by allowing encrypted data to be processed without ever being exposed. This ensures that\n",
      "\n",
      "\n",
      "--------------------------------------------------\n",
      "Epoch 12/20, Loss: 10.0239, grad norm: None, lr: 0.0008470588235294118\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Progress:  60%|██████    | 12/20 [02:15<01:26, 10.86s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prompt: What is FHE?\n",
      "Response: Fully Homomorphic Encryption (FHE) provides a high level of privacy by allowing sensitive information to be processed without ever being exposed. This ensures that\n",
      "\n",
      "\n",
      "--------------------------------------------------\n",
      "Epoch 13/20, Loss: 8.9593, grad norm: None, lr: 0.0007411764705882353\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Progress:  65%|██████▌   | 13/20 [02:25<01:15, 10.75s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prompt: What is FHE?\n",
      "Response: Fully Homomorphic Encryption (FHE) involves encryption of data or computations in a secure environment. This ensures that data or computations remain encrypted\n",
      "\n",
      "\n",
      "--------------------------------------------------\n",
      "Epoch 14/20, Loss: 8.5096, grad norm: None, lr: 0.0006352941176470588\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Progress:  70%|███████   | 14/20 [02:36<01:04, 10.76s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prompt: What is FHE?\n",
      "Response: Fully Homomorphic Encryption (FHE) provides a secure environment for computations on encrypted data. This ensures that sensitive data cannot be decrypted or\n",
      "\n",
      "\n",
      "--------------------------------------------------\n",
      "Epoch 15/20, Loss: 8.4425, grad norm: None, lr: 0.0005294117647058824\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Progress:  75%|███████▌  | 15/20 [02:47<00:54, 10.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prompt: What is FHE?\n",
      "Response: Fully Homomorphic Encryption (FHE) provides a high level of privacy by allowing computations to be performed directly on encrypted data. This ensures that\n",
      "\n",
      "\n",
      "--------------------------------------------------\n",
      "Epoch 16/20, Loss: 7.8181, grad norm: None, lr: 0.0004235294117647059\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Progress:  80%|████████  | 16/20 [02:59<00:44, 11.10s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prompt: What is FHE?\n",
      "Response: Fully Homomorphic Encryption (FHE) is a type of ciphertext that serves as a data store for sensitive data. It has two main components\n",
      "\n",
      "\n",
      "--------------------------------------------------\n",
      "Epoch 17/20, Loss: 6.6628, grad norm: None, lr: 0.0003176470588235294\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Progress:  85%|████████▌ | 17/20 [03:10<00:33, 11.16s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prompt: What is FHE?\n",
      "Response: Fully Homomorphic Encryption (FHE) provides a high level of privacy by allowing computations to be performed directly on encrypted data. This ensures that\n",
      "\n",
      "\n",
      "--------------------------------------------------\n",
      "Epoch 18/20, Loss: 6.7317, grad norm: None, lr: 0.00021176470588235295\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Progress:  90%|█████████ | 18/20 [03:21<00:22, 11.10s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prompt: What is FHE?\n",
      "Response: Fully Homomorphic Encryption (FHE) provides a high level of privacy by allowing computations on encrypted data without ever decrypting it. This ensures\n",
      "\n",
      "\n",
      "--------------------------------------------------\n",
      "Epoch 19/20, Loss: 6.7047, grad norm: None, lr: 0.00010588235294117647\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Progress:  95%|█████████▌| 19/20 [03:31<00:10, 10.86s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prompt: What is FHE?\n",
      "Response: Fully Homomorphic Encryption (FHE) provides a high level of privacy by allowing computations on encrypted data without ever decrypting it. This ensures\n",
      "\n",
      "\n",
      "--------------------------------------------------\n",
      "Epoch 20/20, Loss: 6.2703, grad norm: None, lr: 0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Progress: 100%|██████████| 20/20 [03:42<00:00, 10.75s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prompt: What is FHE?\n",
      "Response: Fully Homomorphic Encryption (FHE) provides a high level of privacy by allowing computations on encrypted data without ever decrypting it. This ensures\n",
      "\n",
      "\n",
      "--------------------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training Progress: 100%|██████████| 20/20 [03:42<00:00, 11.14s/it]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0kAAAIjCAYAAADWYVDIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB2ZUlEQVR4nO3deVwV9f7H8fc5h10BBUVwx11EzT3KfbdyKVusLLXdtLLytt1basvP7N6W22abaWaW2c3KFk3LfcN9w1wQdxYFWQRZ5MzvD4MgQEDhzAFez8eDh56Z7wyf82E68XZmvmMxDMMQAAAAAECSZDW7AAAAAABwJoQkAAAAAMiDkAQAAAAAeRCSAAAAACAPQhIAAAAA5EFIAgAAAIA8CEkAAAAAkAchCQAAAADyICQBAAAAQB6EJABAoY4cOSKLxaI5c+aU6X6nTp0qi8VSpvt0dr1791bv3r3NLqNcXOlxYrFYNHXq1DKtCQCuFCEJQJU1Z84cWSwWbdmyxexSLiknVBT1FRMTY3aJBaSlpWnq1KlauXKl2aXkk7dvLi4u8vPzU6dOnfTYY48pIiLC7PLKVHHHTc5XZQ1vAHAlXMwuAABQMjNnzlT16tULLK9Ro4bjiylGWlqapk2bJkkFfgn/17/+pWeeecaEqi4aMGCA7r77bhmGoaSkJO3cuVOfffaZ3n//fc2YMUNPPPFEmX/PX3/9tcz3WZybbrpJzZo1y3197tw5jR8/XjfeeKNuuumm3OV16tS5ou/TqFEjnT9/Xq6urpe1/fnz5+Xiwq8jAJwLn0oAUEHcfPPNqlWrltllXDEXFxdTfylu0aKFRo8enW/Zq6++qqFDh+rJJ59Uq1atdN1115XJ90pLS5OXl5fc3NzKZH+l0a5dO7Vr1y739ZkzZzR+/Hi1a9euwPvPKz09XW5ubrJaS3axicVikYeHx2XXeSXbAkB54XI7ACjG9u3bNWTIEPn4+Kh69erq16+fNm7cmG9MVlaWpk2bpubNm8vDw0P+/v7q3r27li1bljsmJiZG48aNU/369eXu7q6goCANHz5cR44cueIaY2Nj5eLiknv2Jq/9+/fLYrHo3XffzV12+PBh3XLLLfLz85OXl5euvvpq/fTTT8V+n6LurRk7dqwaN24s6eI9KrVr15YkTZs2Lfeyrpz7Tgq7J+nChQt66aWX1LRpU7m7u6tx48Z67rnnlJGRkW9c48aNdcMNN2jt2rXq2rWrPDw81KRJE82dO7fY2i/F399fX331lVxcXPTKK6/kLs+5JPPvP6OVK1fKYrHku5ywd+/eCg0N1datW9WzZ095eXnpueeey12Xt28523/99dd65ZVXVL9+fXl4eKhfv346dOhQgfree+89NWnSRJ6enuratavWrFlTJvc55dTx1Vdf6V//+pfq1asnLy8vJScnKyEhQZMnT1bbtm1VvXp1+fj4aMiQIdq5c2e+fRR2T9LYsWNVvXp1nTx5UiNGjFD16tVVu3ZtTZ48WdnZ2fm2//s9STnHx6FDhzR27FjVqFFDvr6+GjdunNLS0vJte/78eT366KOqVauWvL29NWzYMJ08eZL7nABcMc4kAcAl7N27Vz169JCPj4+eeuopubq66sMPP1Tv3r21atUqdevWTdLFX+ymT5+u++67T127dlVycrK2bNmibdu2acCAAZKkkSNHau/evXrkkUfUuHFjxcXFadmyZTp27FhuwLiUhISEAstcXFxUo0YN1alTR7169dLXX3+tKVOm5BuzYMEC2Ww23XLLLZIuBqprrrlGaWlpevTRR+Xv76/PPvtMw4YN0zfffKMbb7zxinpWu3ZtzZw5s8ClXXnPavzdfffdp88++0w333yznnzySW3atEnTp0/Xvn37tGjRonxjDx06pJtvvln33nuvxowZo08//VRjx45Vp06d1KZNm8uuu2HDhurVq5dWrFih5ORk+fj4lHof8fHxGjJkiEaNGqXRo0cXeynbq6++KqvVqsmTJyspKUmvvfaa7rzzTm3atCl3zMyZMzVx4kT16NFDjz/+uI4cOaIRI0aoZs2aql+/fqlrLMxLL70kNzc3TZ48WRkZGXJzc1NERIS+++473XLLLQoODlZsbKw+/PBD9erVSxEREapbt+4l95mdna1BgwapW7du+s9//qPly5fr9ddfV9OmTTV+/Phia7r11lsVHBys6dOna9u2bfrkk08UEBCgGTNm5I4ZO3asvv76a9111126+uqrtWrVKl1//fVX3A8AkAEAVdTs2bMNScbmzZuLHDNixAjDzc3NiIyMzF126tQpw9vb2+jZs2fusvbt2xvXX399kfs5e/asIcn497//Xeo6p0yZYkgq9Ktly5a54z788ENDkrF79+5824eEhBh9+/bNfT1p0iRDkrFmzZrcZSkpKUZwcLDRuHFjIzs72zAMw4iKijIkGbNnz84d16tXL6NXr14FahwzZozRqFGj3NenT582JBlTpkwp8v3k2LFjhyHJuO+++/KNmzx5siHJ+P3333OXNWrUyJBkrF69OndZXFyc4e7ubjz55JMFvtffSTImTJhQ5PrHHnvMkGTs3LnTMIy/jpGoqKh841asWGFIMlasWJG7rFevXoYk44MPPiiw37/3LWf71q1bGxkZGbnL//vf/+b7GWZkZBj+/v5Gly5djKysrNxxc+bMMSQV+rMoSmE/k5w6mjRpYqSlpeUbn56ennss5IiKijLc3d2NF198Md+yvx8nY8aMMSTlG2cYhtGhQwejU6dO+Zb9vaac4+Oee+7JN+7GG280/P39c19v3brVkGRMmjQp37ixY8cWeewBQElxuR0AFCE7O1u//vqrRowYoSZNmuQuDwoK0h133KG1a9cqOTlZ0sXJE/bu3auDBw8Wui9PT0+5ublp5cqVOnv27GXV87///U/Lli3L9zV79uzc9TfddJNcXFy0YMGC3GV79uxRRESEbrvtttxlP//8s7p27aru3bvnLqtevboeeOABHTlyxOGzvP3888+SVGDChCeffFKSClwGGBISoh49euS+rl27tlq2bKnDhw9fcS05E2OkpKRc1vbu7u4aN25cicePGzcu3/1KOe8r571s2bJF8fHxuv/++/Pdx3XnnXeqZs2al1VjYcaMGSNPT898y9zd3XPvS8rOzlZ8fLyqV6+uli1batu2bSXa70MPPZTvdY8ePUr8cyps2/j4+Nz/5pYsWSJJevjhh/ONe+SRR0q0fwC4FEISABTh9OnTSktLU8uWLQusa926tex2u44fPy5JevHFF5WYmKgWLVqobdu2+sc//qFdu3bljnd3d9eMGTP0yy+/qE6dOurZs6dee+21Uk3f3bNnT/Xv3z/fV1hYWO76WrVqqV+/fvr6669zly1YsEAuLi75ZjM7evRoke8pZ70jHT16VFarNd9MbJIUGBioGjVqFKinYcOGBfZRs2bNyw6feZ07d06S5O3tfVnb16tXr1STNPz9veQEn5z3kvPe/94bFxeXEl2iWVLBwcEFltntdr355ptq3ry53N3dVatWLdWuXVu7du1SUlJSsfv08PDIvTctR2l+TiXpjdVqLVD733sFAJeDkAQAZaBnz56KjIzUp59+qtDQUH3yySfq2LGjPvnkk9wxkyZN0oEDBzR9+nR5eHjo+eefV+vWrbV9+/Yyq2PUqFE6cOCAduzYIUn6+uuv1a9fvzKbFa+oh8D+/Wb8stz339lstkKXG4ZxxTXs2bNHNpst9xfv0r7fv5+NKU55vpfSKKzu//u//9MTTzyhnj17at68eVq6dKmWLVumNm3ayG63F7vPot5bSTlLbwBUTYQkAChC7dq15eXlpf379xdY98cff8hqtapBgwa5y/z8/DRu3Dh9+eWXOn78uNq1a1dghq2mTZvqySef1K+//qo9e/YoMzNTr7/+epnVPGLECLm5uWnBggXasWOHDhw4oFGjRuUb06hRoyLfU876otSsWVOJiYkFlv/9bE9JA0/O97Pb7QUuVYyNjVViYuIl6ylLx44d06pVqxQWFpZ7Jinn7MXf37OjzrblvPe/z3h34cKFMpkV8VK++eYb9enTR7NmzdKoUaM0cOBA9e/fv9CfvxlyjpuoqKh8ywubHRAASouQBABFsNlsGjhwoL7//vt8v5DGxsZq/vz56t69e+4MaPHx8fm2rV69upo1a5Y7hXVaWprS09PzjWnatKm8vb0LTHN9JWrUqKFBgwbp66+/1ldffSU3NzeNGDEi35jrrrtO4eHh2rBhQ+6y1NRUffTRR2rcuLFCQkKK3H/Tpk31xx9/6PTp07nLdu7cqXXr1uUb5+XlJalguChMzjOJ3nrrrXzL33jjDUlyyGxlCQkJuv3225Wdna1//vOfucubNm0qSVq9enXusuzsbH300UflXpMkde7cWf7+/vr444914cKF3OVffPFFmVxeeCk2m63AWZuFCxfq5MmT5fp9S2rQoEGSpPfffz/f8nfeeceMcgBUMkwBDqDK+/TTT3NvAs/rscce08svv6xly5ape/fuevjhh+Xi4qIPP/xQGRkZeu2113LHhoSEqHfv3urUqZP8/Py0ZcsWffPNN5o4caIk6cCBA+rXr59uvfVWhYSEyMXFRYsWLVJsbGyBMz1F+eabb3InFshrwIAB+aaavu222zR69Gi9//77GjRokGrUqJFv/DPPPKMvv/xSQ4YM0aOPPio/Pz999tlnioqK0v/+979LPkT0nnvu0RtvvKFBgwbp3nvvVVxcnD744AO1adMm94Z66eLlWyEhIVqwYIFatGghPz8/hYaGKjQ0tMA+27dvrzFjxuijjz5SYmKievXqpfDwcH322WcaMWKE+vTpU6L+lNSBAwc0b948GYah5ORk7dy5UwsXLtS5c+f0xhtvaPDgwblj27Rpo6uvvlrPPvusEhIS5Ofnp6+++ipfYClPbm5umjp1qh555BH17dtXt956q44cOaI5c+aoadOmpTpjV1o33HCDXnzxRY0bN07XXHONdu/erS+++CLfJCZm6tSpk0aOHKm33npL8fHxuVOAHzhwQFLpzmYCwN8RkgBUeTNnzix0+dixY9WmTRutWbNGzz77rKZPny673a5u3bpp3rx5uc9IkqRHH31UP/zwg3799VdlZGSoUaNGevnll/WPf/xDktSgQQPdfvvt+u233/T555/LxcVFrVq10tdff62RI0eWqM6ini2zYsWKfCFp2LBh8vT0VEpKSr5Z7XLUqVNH69ev19NPP6133nlH6enpateunRYvXlzsWZvWrVtr7ty5euGFF/TEE08oJCREn3/+uebPn5/vwaqS9Mknn+iRRx7R448/rszMTE2ZMqXQkJQztkmTJpozZ44WLVqkwMBAPfvsswWe+VQWcmYGtFqt8vHxUXBwsMaMGaMHHnig0LNoX3zxhR588EG9+uqrqlGjhu6991716dMn9/lX5W3ixIkyDEOvv/66Jk+erPbt2+uHH37Qo48+Kg8Pj3L7vs8995xSU1M1f/58LViwQB07dtRPP/2kZ555pty+Z2nNnTtXgYGB+vLLL7Vo0SL1799fCxYsUMuWLcu1NwAqP4vBHZAAAFQodrtdtWvX1k033aSPP/7Y7HKcyo4dO9ShQwfNmzdPd955p9nlAKiguCcJAAAnlp6eXuDeoLlz5yohIUG9e/c2pygncf78+QLL3nrrLVmtVvXs2dOEigBUFlxuBwCAE9u4caMef/xx3XLLLfL399e2bds0a9YshYaG6pZbbjG7PFO99tpr2rp1q/r06SMXFxf98ssv+uWXX/TAAw/km3kSAEqLy+0AAHBiR44c0aOPPqrw8PDcySOuu+46vfrqqwoICDC7PFMtW7ZM06ZNU0REhM6dO6eGDRvqrrvu0j//+U+5uPDvwAAuHyEJAAAAAPLgniQAAAAAyIOQBAAAAAB5VPoLdu12u06dOiVvb28eLAcAAABUYYZhKCUlRXXr1r3kw9MrfUg6deoUM9wAAAAAyHX8+HHVr1+/yPWVPiR5e3tLutgIHx8fk6up3LKysvTrr79q4MCBcnV1NbucSo9+Ox49dzx67nj03LHot+PRc8dzpp4nJyerQYMGuRmhKJU+JOVcYufj40NIKmdZWVny8vKSj4+P6f8BVAX02/HouePRc8ej545Fvx2PnjueM/a8uNtwmLgBAAAAAPIgJAEAAABAHoQkAAAAAMiDkAQAAAAAeRCSAAAAACAPQhIAAAAA5EFIAgAAAIA8CEkAAAAAkAchCQAAAADyICQBAAAAQB6EJAAAAADIg5AEAAAAAHkQkgAAAAAgDxezC6gqsu2GwqMSFJeSrgBvD3UN9pPNajG7LAAAAAB/Q0hygCV7ojVtcYSik9JzlwX5emjK0BANDg0ysTIAAAAAf8flduVsyZ5ojZ+3LV9AkqSYpHSNn7dNS/ZEm1QZAAAAgMIQkspRtt3QtMURMgpZl7Ns2uIIZdsLGwEAAADADISkchQelVDgDFJehqTopHSFRyU4rigAAAAAl0RIKkdxKUUHpMsZBwAAAKD8EZLKUYC3R5mOAwAAAFD+CEnlqGuwn4J8PVTURN8WXZzlrmuwnyPLAgAAAHAJhKRyZLNaNGVoiCQVGZSmDA3heUkAAACAEyEklbPBoUGaObqjAn0LXlL34vA2PCcJAAAAcDI8TNYBBocGaUBIoMKjEhSXkq5P10Zp54kknbrEzHcAAAAAzMGZJAexWS0Ka+qv4VfV0/jeTSVJC7ecUFa23eTKAAAAAORFSDJBv9Z1VKu6u86cy9Bv++LMLgcAAABAHoQkE7jarLq5U31J0lebj5lcDQAAAIC8CEkmGdWlgSRp1YHTOpl43uRqAAAAAOQgJJmkca1qCmviL8OQvt583OxyAAAAAPyJkGSiUV0vnk1auOW4su2GydUAAAAAkAhJphrUJlA1vFx1Kildqw+cNrscAAAAACIkmcrD1aabOlycwOHLcCZwAAAAAJwBIclkt/95yd1vf8QpLpmHywIAAABmIySZrHkdb3VqVFPZdkMLt54wuxwAAACgyiMkOYGc6cAXbD4uOxM4AAAAAKYiJDmB69sFydvdRccS0rThcLzZ5QAAAABVGiHJCXi5uWh4h7qSmMABAAAAMBshyUmM6tJQkvTr3lglpGaaXA0AAABQdRGSnERoPV+1reerzGy7vt3GBA4AAACAWQhJTmTUn9OBfxl+TIbBBA4AAACAGQhJTmRY+7rydLUp8nSqthw9a3Y5AAAAQJVESHIi3h6uGto+SBITOAAAAABmISQ5mVFdL07g8PPuaCWdzzK5GgAAAKDqISQ5mQ4NaqhlHW+lZ9n1/Y6TZpcDAAAAVDmEJCdjsVjyTOBwnAkcAAAAAAczNSTNnDlT7dq1k4+Pj3x8fBQWFqZffvkld316eromTJggf39/Va9eXSNHjlRsbKyJFTvGjR3qyc3Fqn3Rydp1IsnscgAAAIAqxdSQVL9+fb366qvaunWrtmzZor59+2r48OHau3evJOnxxx/X4sWLtXDhQq1atUqnTp3STTfdZGbJDlHDy03XhQZKkr7azAQOAAAAgCOZGpKGDh2q6667Ts2bN1eLFi30yiuvqHr16tq4caOSkpI0a9YsvfHGG+rbt686deqk2bNna/369dq4caOZZTtEzgQOP+w4pdSMCyZXAwAAAFQdLmYXkCM7O1sLFy5UamqqwsLCtHXrVmVlZal///65Y1q1aqWGDRtqw4YNuvrqqwvdT0ZGhjIyMnJfJycnS5KysrKUlVVxZovrWN9bwf5eiopP03fbjuvWzvXNLqlYOf2tSH2uyOi349Fzx6PnjkfPHYt+Ox49dzxn6nlJa7AYJs8MsHv3boWFhSk9PV3Vq1fX/Pnzdd1112n+/PkaN25cvsAjSV27dlWfPn00Y8aMQvc3depUTZs2rcDy+fPny8vLq1zeQ3n57aRFPxyzqVF1Q0+0zTa7HAAAAKBCS0tL0x133KGkpCT5+PgUOc70M0ktW7bUjh07lJSUpG+++UZjxozRqlWrLnt/zz77rJ544onc18nJyWrQoIEGDhx4yUY4o27nMvTLf1br6DmpScceahXobXZJl5SVlaVly5ZpwIABcnV1NbucSo9+Ox49dzx67nj03LHot+PRc8dzpp7nXGVWHNNDkpubm5o1ayZJ6tSpkzZv3qz//ve/uu2225SZmanExETVqFEjd3xsbKwCAwOL3J+7u7vc3d0LLHd1dTX9h1JagTVdNSCkjn7eHaNvtp3StOGhZpdUIhWx1xUZ/XY8eu549Nzx6Llj0W/Ho+eO5ww9L+n3d7rnJNntdmVkZKhTp05ydXXVb7/9lrtu//79OnbsmMLCwkys0LFGdbk4gcOi7SeVnsUldwAAAEB5M/VM0rPPPqshQ4aoYcOGSklJ0fz587Vy5UotXbpUvr6+uvfee/XEE0/Iz89PPj4+euSRRxQWFlbkpA2VUfdmtVS/pqdOnD2vn3dH66aOzj+BAwAAAFCRmRqS4uLidPfddys6Olq+vr5q166dli5dqgEDBkiS3nzzTVmtVo0cOVIZGRkaNGiQ3n//fTNLdjir1aLbOjfQ68sO6Kvw44QkAAAAoJyZGpJmzZp1yfUeHh5677339N577zmoIud0S+cGenP5AYUfSdChuBQ1C3DuCRwAAACAiszp7klCQYG+HurbKkCS9FX4cZOrAQAAACo3QlIFkTOBw/+2nVDGBSZwAAAAAMoLIamC6N2ytur4uOtsWpZ+3RtrdjkAAABApUVIqiBcbFbd2rmBJOmrzcdMrgYAAACovAhJFcitnRvIYpHWHYrX0fhUs8sBAAAAKiVCUgXSwM9L3ZvVkiQt2MwEDgAAAEB5ICRVMLd3vTiBw8KtJ5SVbTe5GgAAAKDyISRVMP1b15F/NTedTsnQ73/EmV0OAAAAUOkQkioYNxerbu5UX5L0VTgTOAAAAABljZBUAd3W5eIsd6sOnNapxPMmVwMAAABULoSkCqhJ7erqFuwnuyF9vYUJHAAAAICyREiqoHImcPh683Fl2w2TqwEAAAAqD0JSBTU4NFC+nq46lZSu1QdPm10OAAAAUGkQkiooD1ebbuxQTxITOAAAAABliZBUgeVccvfbvjjFpaSbXA0AAABQORCSKrCWgd7q0LCGLtgNfbP1hNnlAAAAAJUCIamCu73LxbNJCzYfl50JHAAAAIArRkiq4G5oH6Tq7i46Gp+mjYfjzS4HAAAAqPAISRWcl5uLhl1VV5L05WaemQQAAABcKUJSJZBzyd3SPTFKSM00uRoAAACgYiMkVQJt6/uqTV0fZWbb9e02JnAAAAAArgQhqZIY9ed04F9tPi7DYAIHAAAA4HIRkiqJ4VfVlaerTYfizmnr0bNmlwMAAABUWISkSsLHw1XXtwuSJH0ZzgQOAAAAwOUiJFUit3dtIEn6afcpJZ3PMrkaAAAAoGIiJFUiHRvWVPOA6krPsuuHHSfNLgcAAACokAhJlYjFYsmdwOHLcCZwAAAAAC4HIamSualDPbnZrIqITtbuk0lmlwMAAABUOISkSqZmNTcNDg2UxAQOAAAAwOUgJFVCo/6cwOGHHSeVmnHB5GoAAACAioWQVAmFNfFXY38vpWZm68ddp8wuBwAAAKhQCEmVkMVi0W1d/prAAQAAAEDJEZIqqZs71ZeL1aIdxxP1R0yy2eUAAAAAFQYhqZKq7e2u/q3rSJK+4mwSAAAAUGKEpEosZwKHb7edUHpWtsnVAAAAABUDIakS69G8turV8FRy+gX9sifa7HIAAACACoGQVInZrBbd2vni2SQmcAAAAABKhpBUyd3apb6sFik8KkGRp8+ZXQ4AAADg9AhJlVyQr6d6twyQJC3YzNkkAAAAoDiEpCpgVJeLl9z9b+sJZV6wm1wNAAAA4NwISVVA31YBCvB2V3xqppZFxJpdDgAAAODUCElVgIvNqls615ckfbX5mMnVAAAAAM6NkFRF3Na5oSRpzcEzOp6QZnI1AAAAgPMiJFURDf291L1ZLUlM4AAAAABcCiGpChnV9eIEDgu3HteFbCZwAAAAAApDSKpCBoTUkV81N8UmZ2jF/tNmlwMAAAA4JUJSFeLuYtPIjvUkSV+FM4EDAAAAUBhCUhVzW5eLEzis2B+n6KTzJlcDAAAAOB9CUhXTLKC6ujb2k92QFm45YXY5AAAAgNMhJFVBORM4LNh8XHa7YXI1AAAAgHMhJFVB17UNko+Hi04mnteaQ2fMLgcAAABwKoSkKsjD1aYbO1ycwOG93w/q+x0ntSEyXtmcVQIAAADkYnYBMEcDPy9JUviRswo/claSFOTroSlDQzQ4NMjM0gAAAABTcSapClqyJ1qv/LSvwPKYpHSNn7dNS/ZEm1AVAAAA4BwISVVMtt3QtMURKuzCupxl0xZHcOkdAAAAqixCUhUTHpWg6KT0ItcbkqKT0hUeleC4ogAAAAAnQkiqYuJSig5IlzMOAAAAqGwISVVMgLdHmY4DAAAAKhtCUhXTNdhPQb4eslxiTJCvh7oG+zmsJgAAAMCZEJKqGJvVoilDQySpyKD07JBWslkvFaMAAACAyouQVAUNDg3SzNEdFeib/5K6nFy0NzrZhKoAAAAA58DDZKuowaFBGhASqPCoBMWlpCvA20Mp57P0wLyt+mRNlIa2q6vQer5mlwkAAAA4HCGpCrNZLQpr6p9v2Q3tgvTjrmg99c0ufT/xWrnaONkIAACAqoXfgJHP1GFtVMPLVRHRyfp4zWGzywEAAAAcjpCEfGpVd9e/rr84scNbyw8q6kyqyRUBAAAAjkVIQgEjO9ZTj+a1lHnBrmf+t0t2u2F2SQAAAIDDEJJQgMVi0f/d2FaerjZtikrQV5uPm10SAAAA4DCEJBSqgZ+XJg9qKUma/vM+xSSlm1wRAAAA4BiEJBRp7DWN1b5BDaVkXNDz3++RYXDZHQAAACo/QhKKZLNaNGNkW7lYLVoWEatf9sSYXRIAAABQ7ghJuKRWgT56uHdTSdIL3+9VYlqmyRUBAAAA5cvUkDR9+nR16dJF3t7eCggI0IgRI7R///58Y3r37i2LxZLv66GHHjKp4qppQt9mahZQXWfOZeiVn/aZXQ4AAABQrkwNSatWrdKECRO0ceNGLVu2TFlZWRo4cKBSU/M/m+f+++9XdHR07tdrr71mUsVVk7uLTTNGtpXFIi3cekJrD54xuyQAAACg3LiY+c2XLFmS7/WcOXMUEBCgrVu3qmfPnrnLvby8FBgY6OjykEenRn666+pGmrvhqJ5dtEu/TuolTzeb2WUBAAAAZc7UkPR3SUlJkiQ/P798y7/44gvNmzdPgYGBGjp0qJ5//nl5eXkVuo+MjAxlZGTkvk5OTpYkZWVlKSsrq5wqrxoe79dUyyJidTzhvP6zdJ+eGdwy3/qc/tJnx6DfjkfPHY+eOx49dyz67Xj03PGcqeclrcFiOMm8zna7XcOGDVNiYqLWrl2bu/yjjz5So0aNVLduXe3atUtPP/20unbtqm+//bbQ/UydOlXTpk0rsHz+/PlFBiuU3N6zFn30h00WGXq8bbYaVTe7IgAAAKBk0tLSdMcddygpKUk+Pj5FjnOakDR+/Hj98ssvWrt2rerXr1/kuN9//139+vXToUOH1LRp0wLrCzuT1KBBA505c+aSjUDJPbFwlxbvilGrOtX17fir5Wq7eGtbVlaWli1bpgEDBsjV1dXkKis/+u149Nzx6Lnj0XPHot+OR88dz5l6npycrFq1ahUbkpzicruJEyfqxx9/1OrVqy8ZkCSpW7duklRkSHJ3d5e7u3uB5a6urqb/UCqLqcNCtfZQvP6IPadP1x/TxL7N862n145Fvx2PnjsePXc8eu5Y9Nvx6LnjOUPPS/r9TZ3dzjAMTZw4UYsWLdLvv/+u4ODgYrfZsWOHJCkoKKicq0NR/Ku764WhIZKkt387pENx50yuCAAAACg7poakCRMmaN68eZo/f768vb0VExOjmJgYnT9/XpIUGRmpl156SVu3btWRI0f0ww8/6O6771bPnj3Vrl07M0uv8kZcVU+9W9ZWZrZdz367S3a7U1y1CQAAAFwxU0PSzJkzlZSUpN69eysoKCj3a8GCBZIkNzc3LV++XAMHDlSrVq305JNPauTIkVq8eLGZZUOSxWLRyyNC5eVm0+YjZ/VF+DGzSwIAAADKhKn3JBU3Z0SDBg20atUqB1WD0qpf00tPDWqpqYsjNOOXP9SrmV/xGwEAAABOztQzSaj47gprrA4Na+hcxgVNWRwh55grEQAAALh8hCRcEZvVohkj28nVZtGK/We0Pd5idkkAAADAFSEk4Yq1qOOtCX2aSZL+F2XV2bRMkysCAAAALh8hCWXi4d7N1Dygms5dsGj6L/vNLgcAAAC4bIQklAk3F6teGdFGFhlatCNaqw6cNrskAAAA4LIQklBmOjSooR6BF2dueO7b3UrNuGByRQAAAEDpEZJQpm5oaFe9Gh46mXher/96wOxyAAAAgFIjJKFMudukl4aFSJJmr4/S9mNnTa4IAAAAKB1CEspcj+a1dFOHejIM6Zn/7VbmBbvZJQEAAAAlRkhCufjXDSHyq+am/bEp+mBVpNnlAAAAACVGSEK58KvmpilDL1529+7vh3QoLsXkigAAAICSISSh3AxrX1d9WwUoM9uup/+3W3a7YXZJAAAAQLEISSg3FotFL48IVXV3F209elafbzxqdkkAAABAsQhJKFd1a3jq6cEtJUmvLflDJxPPm1wRAAAAcGmEJJS7O7s1UudGNZWama1/Ldotw+CyOwAAADgvQhLKndVq0asj28nNZtWK/af1w85TZpcEAAAAFImQBIdoFlBdj/RtJkmatjhCCamZJlcEAAAAFI6QBId5sFdTtQr0VkJqpl5cvNfscgAAAIBCEZLgMG4uVr06sp2sFum7Hae0Yn+c2SUBAAAABRCS4FBXNaihcdcGS5L++e1uncu4YHJFAAAAQH6EJDjckwNbqIGfp04lpes/S/ebXQ4AAACQDyEJDufl5qLpN7aTJH224Yi2Hj1rckUAAADAXwhJMEX35rV0c6f6Mgzp6f/tUsaFbLNLAgAAACQRkmCif13fWrWqu+lQ3Dm9vyLS7HIAAAAASYQkmKiGl5umDmsjSXp/5SEdiE0xuSIAAACAkASTXd82SP1b11FWtqGnvtmlbLthdkkAAACo4lzMLgBVm8Vi0csjQrXpcLx2HE/U7HVRalPXV3Ep6Qrw9lDXYD/ZrBazywQAAEAVQkiC6QJ9PfTMda30z0V79PJP+/KtC/L10JShIRocGmRSdQAAAKhquNwOTqGmp1uhy2OS0jV+3jYt2RPt4IoAAABQVRGSYLpsu6GXfooodF3OHUrTFkdwvxIAAAAcgpAE04VHJSg6Kb3I9Yak6KR0hUclOK4oAAAAVFmEJJguLqXogHQ54wAAAIArQUiC6QK8Pcp0HAAAAHAlCEkwXddgPwX5euhSE30H+V6cDhwAAAAob4QkmM5mtWjK0BBJKjIoNfKrdskQBQAAAJQVQhKcwuDQIM0c3VGBvvkvqavp5SqrRdoYFa/pv+yTYTDDHQAAAMoXD5OF0xgcGqQBIYEKj0pQXEq6ArwvXmK3aPtJTV64Ux+viZJfNXeN793U7FIBAABQiRGS4FRsVovCmvrnW3Zzp/o6m5qpV37epxlL/pBfNVfd1qWhSRUCAACgsuNyO1QI9/dsood6XTyD9Oy3u7VkT4zJFQEAAKCyIiShwnh6cEvd2rm+7Ib06FfbtSEy3uySAAAAUAkRklBhWCwW/d+NbTUwpI4yL9h1/9wt2nMyyeyyAAAAUMkQklChuNisevv2DuoW7KdzGRc05tNwRZ1JNbssAAAAVCKEJFQ4Hq42fTyms0KCfBSfmqm7Zm1SbHK62WUBAACgkiAkoULy8XDVZ/d0VWN/L504e153zwpXUlqW2WUBAACgEiAkocKq7e2uz+/tpgBvd+2PTdG9n23W+cxss8sCAABABUdIQoXWwM9Lc+/tKh8PF205elYPf7FVWdl2s8sCAABABUZIQoXXKtBHn47tIg9Xq1bsP62nvtklu90wuywAAABUUIQkVAqdG/vp/Ts7yma1aNH2k3r5p30yDIISAAAASo+QhEqjb6s6+s8t7SRJn66L0vsrI02uCAAAABURIQmVyo0d6uv5G0IkSf9eul/zNx0zuSIAAABUNIQkVDr3dg/WhD5NJUn/+m63ftkdbXJFAAAAqEgISaiUJg9sqdu7NpTdkB77aofWHzpjdkkAAACoIAhJqJQsFoteHhGqIaGBysy26/65W7TrRKLZZQEAAKACICSh0rJZLXpr1FW6pqm/UjOzNXb2ZkWePmd2WQAAAHByhCRUau4uNn10d2e1reerhNRM3T0rXDFJ6WaXBQAAACdGSEKlV93dRXPGdVGTWtV0MvG87pq1SYlpmWaXBQAAACdFSEKV4F/dXXPv7ao6Pu46GHdO4+ZsVlrmBbPLAgAAgBMiJKHKqF/TS5/f202+nq7afixR4+dtU+YFu9llAQAAwMmUOiQdP35cJ06cyH0dHh6uSZMm6aOPPirTwoDy0KKOtz4d20WerjatOnBakxfulN1umF0WAAAAnEipQ9Idd9yhFStWSJJiYmI0YMAAhYeH65///KdefPHFMi8QKGudGtXUzNEd5WK16Iedp/TijxEyDIISAAAALip1SNqzZ4+6du0qSfr6668VGhqq9evX64svvtCcOXPKuj6gXPRuGaDXb20vSZqz/oje+f2QyRUBAADAWZQ6JGVlZcnd3V2StHz5cg0bNkyS1KpVK0VHR5dtdUA5Gn5VPU0dGiJJemPZAX2+8ajJFQEAAMAZlDoktWnTRh988IHWrFmjZcuWafDgwZKkU6dOyd/fv8wLBMrT2GuD9WjfZpKkF77fox93nTK5IgAAAJit1CFpxowZ+vDDD9W7d2/dfvvtat/+4iVLP/zwQ+5leEBF8viAFrqzW0MZhvT4gh1ac/C02SUBAADARC6l3aB37946c+aMkpOTVbNmzdzlDzzwgLy8vMq0OMARLBaLXhweqsTzWfppV7Qe/Hyr5t9/ta5qUMPs0gAAAGCCUp9JOn/+vDIyMnID0tGjR/XWW29p//79CggIKPMCAUewWS1649b26t6sltIyszVudrgOxZ0zuywAAACYoNQhafjw4Zo7d64kKTExUd26ddPrr7+uESNGaObMmWVeIOAo7i42fXhXJ7Wv76uzaVm6e9YmnUo8b3ZZAAAAcLBSh6Rt27apR48ekqRvvvlGderU0dGjRzV37ly9/fbbZV4g4EjV3F00e1xXNa1dTaeS0nX3p+E6k5KhDZHx+n7HSW2IjFc2D58FAACo1Ep9T1JaWpq8vb0lSb/++qtuuukmWa1WXX311Tp6lCmUUfH5VXPT3Hu76eaZ63Uo7pzCXv1NWdl/BaMgXw9NGRqiwaFBJlYJAACA8lLqM0nNmjXTd999p+PHj2vp0qUaOHCgJCkuLk4+Pj5lXiBghno1PPVAzyaSlC8gSVJMUrrGz9umJXt4LhgAAEBlVOqQ9MILL2jy5Mlq3LixunbtqrCwMEkXzyp16NChzAsEzJBtN/TR6sOFrsuJTNMWR3DpHQAAQCVU6pB0880369ixY9qyZYuWLl2au7xfv3568803S7Wv6dOnq0uXLvL29lZAQIBGjBih/fv35xuTnp6uCRMmyN/fX9WrV9fIkSMVGxtb2rKBUgmPSlB0UnqR6w1J0UnpCo9KcFxRAAAAcIhShyRJCgwMVIcOHXTq1CmdOHFCktS1a1e1atWqVPtZtWqVJkyYoI0bN2rZsmXKysrSwIEDlZqamjvm8ccf1+LFi7Vw4UKtWrVKp06d0k033XQ5ZQMlFpdSdEDKKya5ZOMAAABQcZR64ga73a6XX35Zr7/+us6du/gcGW9vbz355JP65z//Kau15LlryZIl+V7PmTNHAQEB2rp1q3r27KmkpCTNmjVL8+fPV9++fSVJs2fPVuvWrbVx40ZdffXVpS0fKJEAb48SjXvtlz9kkTS0fV3ZrJbyLQoAAAAOUeqQ9M9//lOzZs3Sq6++qmuvvVaStHbtWk2dOlXp6el65ZVXLruYpKQkSZKfn58kaevWrcrKylL//v1zx7Rq1UoNGzbUhg0bCg1JGRkZysjIyH2dnJwsScrKylJWVtZl14bi5fS3MvS5Q31vBfq4KzY5Q0XddWSRFJ2crkkLdujt3w7qkT5NNCQ00GFhqTL1u6Kg545Hzx2PnjsW/XY8eu54ztTzktZgMQyjVHee161bVx988IGGDRuWb/n333+vhx9+WCdPnizN7nLZ7XYNGzZMiYmJWrt2rSRp/vz5GjduXL7QI128tK9Pnz6aMWNGgf1MnTpV06ZNK7B8/vz58vLyuqzaUDXtjLfo0wM5Z0bzBp+L/8nc1cyuhAxpxSmr0rIvrg/0NDS4vl3t/Q1xYgkAAMC5pKWl6Y477lBSUtIlZ+Yu9ZmkhISEQu89atWqlRISLv8m9gkTJmjPnj25AelyPfvss3riiSdyXycnJ6tBgwYaOHAgU5SXs6ysLC1btkwDBgyQq6ur2eVcseskddwbq5d//kMxyX8F9SBfD/1zSCsNalNHkpSSfkFzNx7Tp+uOKOb8Bc05aFOLpOqa2KeJBoXUkbWc0lJl63dFQM8dj547Hj13LPrtePTc8Zyp5zlXmRWn1CGpffv2evfdd/X222/nW/7uu++qffv2pd2dJGnixIn68ccftXr1atWvXz93eWBgoDIzM5WYmKgaNWrkLo+NjVVgYGCh+3J3d5e7u3uB5a6urqb/UKqKytTrG66qryHt6ik8KkFxKekK8PZQ12C/fJfU+bm6atKAlrqnRxN9ujZKs9ZG6UDcOT26YJdaBXprUv8WGtSmjiyW8glLlanfFQU9dzx67nj03LHot+PRc8dzhp6X9PuXOiS99tpruv7667V8+fLcZyRt2LBBx48f188//1yqfRmGoUceeUSLFi3SypUrFRwcnG99p06d5Orqqt9++00jR46UJO3fv1/Hjh3L/d5AebNZLQpr6l/sOB8PV03q30Ljrg3WrLVRmr02Sn/EpOiheVsVEuSjSf2ba0BI+YUlAAAAlI1STwHeq1cvHThwQDfeeKMSExOVmJiom266Sfv371ePHj1Kta8JEyZo3rx5mj9/vry9vRUTE6OYmBidP39ekuTr66t7771XTzzxhFasWKGtW7dq3LhxCgsLY2Y7OC1fT1c9MaCF1jzdRxP7NFM1N5siopP1wOdbNezddfptX6xKeSsgAAAAHKjUZ5Kki5M3/H0WuxMnTuiBBx7QRx99VOL9zJw5U5LUu3fvfMtnz56tsWPHSpLefPNNWa1WjRw5UhkZGRo0aJDef//9yykbcKgaXm6aPKil7u0erI/XHNac9Ue0+2SS7v1si9rX99Wk/i3Uu2VtziwBAAA4mct6mGxh4uPjNWvWrFJtYxhGoV85AUmSPDw89N577ykhIUGpqan69ttvi7wfCXBGNau56anBrbT26b56qFdTebratPNEksbN2awb31+vlfvjOLMEAADgRMosJAG4NL9qbnpmSCutebqPHujZRB6uVu04nqixszdr5Mz1WnPwNGEJAADACRCSAAerVd1dz13XWmue6qv7ugfL3cWqbccSddescN3ywQatO3SGsAQAAGAiQhJgktre7vrXDSFa81Qfjbu2sdxcrNpy9Kzu/GSTbvtwozZExptdIgAAQJVU4okbbrrppkuuT0xMvNJagCopwMdDU4a20UO9mmrmykjNDz+m8CMJuv3jjbq6iZ8e799C3Zrkn4I8225oU1SCtp6xyD8qQWHNAvI9uwkAAACXr8QhydfXt9j1d9999xUXBFRVdXw8NHXYxbD0/spD+ir8uDYeTtBtH23UNU399fiAFurS2E9L9kRr2uIIRSelS7Jp7sEtCvL10JShIRocGmT22wAAAKjwShySZs+eXZ51APhToK+HXhweqod6NdV7Kw7p6y3HtT4yXusjN6h1kLf2RacU2CYmKV3j523TzNEdCUoAAABXiHuSACdVt4anXrmxrVZM7q3buzaUzaJCA5Ik5UzzMG1xhLLtTPoAAABwJQhJgJOrX9NL029qqzdvu+qS4wxJ0UnpCo9KcEhdAAAAlRUhCaggSnp+KC4lvVzrAAAAqOwISUAFEeDtUabjAAAAUDhCElBBdA32U5Cvhy410Xet6m7qGuznsJoAAAAqoxLNbvfDDz+UeIfDhg277GIAFM1mtWjK0BCNn7dNFhV++V1qxgXtPpmkqxrUcHB1AAAAlUeJQtKIESNKtDOLxaLs7OwrqQfAJQwODdLM0R3zPCfpokAfd3m5u+jw6VSN/mSTPh3bhTNKAAAAl6lEIclut5d3HQBKaHBokAaEBGrDoTj9umaTBvboprBmAUrPytZ9n23RhsPxGvNpuD6+u7O6N69ldrkAAAAVDvckARWQzWpRt2A/daplqFuwn2xWi6q5u2j2uC7q3bK2zmdl657PNuv3P2LNLhUAAKDCKdGZpL9LTU3VqlWrdOzYMWVmZuZb9+ijj5ZJYQBKz8PVpg/v6qRHv9yupXtj9eDnW/X2qA4a0jbI7NIAAAAqjFKHpO3bt+u6665TWlqaUlNT5efnpzNnzsjLy0sBAQGEJMBk7i42vXtHRz359U79sPOUJn65Xa9fsGtEh3pmlwYAAFAhlPpyu8cff1xDhw7V2bNn5enpqY0bN+ro0aPq1KmT/vOf/5RHjQBKydVm1Zu3XaVbOtVXtt3Q41/v0Ffhx8wuCwAAoEIodUjasWOHnnzySVmtVtlsNmVkZKhBgwZ67bXX9Nxzz5VHjQAug81q0YyR7XTX1Y1kGNIz3+7WnHVRZpcFAADg9EodklxdXWW1XtwsICBAx45d/NdpX19fHT9+vGyrA3BFrFaLXhzeRvf3CJYkTV0coQ9WRZpcFQAAgHMr9T1JHTp00ObNm9W8eXP16tVLL7zwgs6cOaPPP/9coaGh5VEjgCtgsVj03HWt5enmord/O6hXf/lD5zOzNal/c1ksFrPLAwAAcDqlPpP0f//3fwoKujhT1iuvvKKaNWtq/PjxOn36tD788MMyLxDAlbNYLHpiQAs9NbilJOm/f4YlwzBMrgwAAMD5lPpMUufOnXP/HhAQoCVLlpRpQQDKz8O9m8nT1aZpiyP04erDOp+VralD28hq5YwSAABAjlKfSerbt68SExMLLE9OTlbfvn3LoiYA5WjctcH6vxvbymKR5m44qme/3a1sO2eUAAAAcpQ6JK1cubLAA2QlKT09XWvWrCmTogCUrzu6NdTrt7SX1SIt2HJcT3y9Qxey7WaXBQAA4BRKfLndrl27cv8eERGhmJiY3NfZ2dlasmSJ6tXjYZVARXFTx/pyd7Hpsa+26/sdp5SRZdfbt3eQm0up/+0EAACgUilxSLrqqqtksVhksVgKvazO09NT77zzTpkWB6B8Xd8uSB6uVo2ft01L9sbowc+3aOboTvJwtZldGgAAgGlKHJKioqJkGIaaNGmi8PBw1a5dO3edm5ubAgICZLPxixVQ0fRrXUezxnbW/XO3aMX+07pnzmZ9MqazvNxKPa8LAABApVDi34IaNWokSbLbuW8BqGx6NK+tz8Z11T1zNmt9ZLzunhWu2eO6yNvD1ezSAAAAHO6ybj6IjIzUI488ov79+6t///569NFHFRkZWda1AXCgbk38Ne++bvLxcNGWo2d15yeblJhWcJIWAACAyq7UIWnp0qUKCQlReHi42rVrp3bt2mnTpk1q06aNli1bVh41AnCQDg1rav79V6uml6t2nUjSqI826sy5DLPLAgAAcKhSh6RnnnlGjz/+uDZt2qQ33nhDb7zxhjZt2qRJkybp6aefLo8aAThQaD1fLXgwTLW93fVHTIpu+3CDYpPTzS4LAADAYUodkvbt26d77723wPJ77rlHERERZVIUAHO1qOOtrx8MU11fD0WeTtWtH27QibNpZpcFAADgEKUOSbVr19aOHTsKLN+xY4cCAgLKoiYATiC4VjUteDBMDfw8dTQ+Tbd9uFFHzqSaXRYAAEC5K3FIevHFF5WWlqb7779fDzzwgGbMmKE1a9ZozZo1evXVV/Xggw/q/vvvL89aAThYAz8vLXzwGjWpXU0nE8/r1g836GBsitllAQAAlKsSh6Rp06bp3Llzev755/XCCy/onXfeUa9evdSrVy+9++67mjp1qv71r3+VZ60ATBDo66EFD4SpVaC34lIydNtHG7X3VJLZZQEAAJSbEockwzAkSRaLRY8//rhOnDihpKQkJSUl6cSJE3rsscdksVjKrVAA5qnt7a4v779abev5KiE1U7d/tFE7jieaXRYAAEC5KNU9SX8PQd7e3vL29i7TggA4p5rV3PTF/d3UqVFNJadf0OhPNmnzkQSzywIAAChzpQpJLVq0kJ+f3yW/AFRePh6umntPV13dxE/nMi7o7lnhWnfojNllAQAAlCmX0gyeNm2afH19y6sWABVANXcXzRnXVQ9+vlWrDpzWuDmb9cHojurbqo6y7YbCoxIUl5KuAG8PdQ32k83KZbgAAKBiKVVIGjVqFNN8A5CHq00f3d1JE+dv17KIWD34+VaNu6axFu+KVnTSXw+eDfL10JShIRocGmRitQAAAKVT4svtmJQBQF7uLja9f2dH3dAuSFnZhj5aE5UvIElSTFK6xs/bpiV7ok2qEgAAoPRKPbsdAORwtVn1xq1XydPVVuj6nE+NaYsjlG3nMwQAAFQMJQ5JdrudS+0AFLD16Fmdz8oucr0hKTopXeFRzIQHAAAqhlLNbgcAfxeXkl78oFKMAwAAMBshCcAVCfD2KNNxAAAAZiMkAbgiXYP9FOTroUtN7eLr6aoujWs6rCYAAIArQUgCcEVsVoumDA2RpCKDUtL5LP3jm11Ky7zguMIAAAAuEyEJwBUbHBqkmaM7KtA3/yV1Qb4euqlDPdmsFi3aflIj3lunyNPnTKoSAACgZEr1MFkAKMrg0CANCAlUeFSC4lLSFeDtoa7BfrJZLbqtSwNN/HK7DsSe07B31mrGze10Q7u6ZpcMAABQKM4kASgzNqtFYU39Nfyqegpr6i+b9eIFeN2a+OunR7vr6iZ+Ss3M1sT52zX1h73KvGA3uWIAAICCCEkAHCLA20Pz7u2m8b2bSpLmrD+i2z7aoFOJ502uDAAAID9CEgCHcbFZ9fTgVvrk7s7y8XDR9mOJuv7tNVp94LTZpQEAAOQiJAFwuP4hdfTjIz0UWs9HZ9OyNGZ2uN5afkDZdsPs0gAAAAhJAMzR0N9L3zx0jW7v2lCGIb21/KDGzg5XQmqm2aUBAIAqjpAEwDQerjZNv6mtXr+lvTxcrVpz8Iyuf3uNth07a3ZpAACgCiMkATDdyE719d2EaxVcq5qik9J124cbNGddlAyDy+8AAIDjEZIAOIVWgT76YeK1uq5toLKyDU1dHKFHvtyucxkXzC4NAABUMYQkAE7D28NV793RUS/cECIXq0U/7orWsHfX6kBsitmlAQCAKoSQBMCpWCwW3dM9WAsevFqBPh46fDpVw99dp++2nzS7NAAAUEUQkgA4pU6N/PTTo93VvVktnc/K1qQFO/TPRbuVcSHb7NIAAEAlR0gC4LT8q7vrs3u66tF+zWWxSF9sOqZbPtig4wlpZpcGAAAqMUISAKdms1r0xIAWmj22i2p4uWrXiSTd8M5a/f5HrNmlAQCASoqQBKBC6N0yQD892kPtG9RQ0vks3TNni/6zdL+y7UwTDgAAyhYhCUCFUa+Gp75+8GrdHdZIkvTuikO6a9YmnTmXYXJlAACgMiEkAahQ3F1senF4qP476ip5udm0PjJe17+9RpuPJJhdGgAAqCQISQAqpOFX1dMPE69Vs4Dqik3O0KiPNuqTNYdlGFx+BwAArgwhCUCF1SzAW99PuFbD2tdVtt3Qyz/t0/h525ScnmV2aQAAoAIjJAGo0Kq5u+i/o67SS8PbyNVm0ZK9MRr2zlpFnEqWJGXbDW2IjNf3O05qQ2Q8Ez0AAIBiuZhdAABcKYvForvCGqtt/Rqa8MU2HYlP043vr9NtXRpoWUSsopPSc8cG+XpoytAQDQ4NMrFiAADgzDiTBKDSuKpBDf34SHf1bllbGRfsmrvhaL6AJEkxSekaP2+bluyJNqlKAADg7EwNSatXr9bQoUNVt25dWSwWfffdd/nWjx07VhaLJd/X4MGDzSkWQIVQs5qbPr6rs6q7F36iPOdiu2mLI7j0DgAAFMrUkJSamqr27dvrvffeK3LM4MGDFR0dnfv15ZdfOrBCABXRlqNndS7jQpHrDUnRSekKj2LacAAAUJCp9yQNGTJEQ4YMueQYd3d3BQYGOqgiAJVBXEp68YMkvbbkD93bI1i9WwYUeeYJAABUPU7/W8HKlSsVEBCgmjVrqm/fvnr55Zfl7+9f5PiMjAxlZGTkvk5OvjjDVVZWlrKymBa4POX0lz47Bv0umr9XyT7ath9P1MT52+XmYtW1Tf00oHUd9WtVW37V3AodT88dj547Hj13LPrtePTc8Zyp5yWtwWI4yZMXLRaLFi1apBEjRuQu++qrr+Tl5aXg4GBFRkbqueeeU/Xq1bVhwwbZbLZC9zN16lRNmzatwPL58+fLy8urvMoH4ETshjRtm02JmZJkKWSEoequUtdahnadtehM+l9jLDLU1MdQez9Dbf0M1XR3VNUAAKC8paWl6Y477lBSUpJ8fHyKHOfUIenvDh8+rKZNm2r58uXq169foWMKO5PUoEEDnTlz5pKNwJXLysrSsmXLNGDAALm6uppdTqVHvy9t6d5YPfLVTkl/TdYg/RWZ3hnVXoPa1JFhGDoYd06/RsTp14g47YtJybefdvV8NDCkjga0DlCDGm703ME4zh2PnjsW/XY8eu54ztTz5ORk1apVq9iQ5PSX2+XVpEkT1apVS4cOHSoyJLm7u8vdveA//bq6upr+Q6kq6LVj0e/C3XBVfbm42DRtcUS+acADC3lOUpv6fmpT30+PD2yl4wlpWro3Rkv3xmjL0bPadTJZu04m6z/LDqpZ7Wpq4mZVo9Pn1b6hpyyWws5SoTxwnDsePXcs+u149NzxnKHnJf3+FSoknThxQvHx8QoK4iGQAIo3ODRIA0ICFR6VoLiUdAV4e6hrsJ9s1qLDTQM/L93Xo4nu69FEcSnpWhYRq6V7Y7X+0BkdOp2qQ7Lq15kbVa+Gpwa1CdSgNnXUufGl9wkAACoWU0PSuXPndOjQodzXUVFR2rFjh/z8/OTn56dp06Zp5MiRCgwMVGRkpJ566ik1a9ZMgwYNMrFqABWJzWpRWNOiJ3u5lABvD93ZrZHu7NZISeeztGxvtOb+vlMHU1x0MvG8Pl0XpU/XRcm/mpsGtqmjgW0CdU1Tf7m7FH7PZI5su1Gq4AYAABzL1JC0ZcsW9enTJ/f1E088IUkaM2aMZs6cqV27dumzzz5TYmKi6tatq4EDB+qll14q9HI6AChPvp6uGt4+SK4nt6tP/z7acCRRS/fGaHlErOJTM/Vl+HF9GX5c3u4u6tMqQINDA9WrRW1V+9vU4kv2RBe4BDCokEsAAQCAeUwNSb1799al5o1YunSpA6sBgJLxdLP9ealdoLKy7dp4OP7P+5hidTolQz/sPKUfdp6Su4tVPZrX1uDQQPVvHaCNh+M1ft42/f1TLyYpXePnbdPM0R0JSgAAOIEKdU8SADgbV9vFINSjeW29OCxU249fPMO0ZE+MjiWkafm+WC3fFyurRXKxWgsEJOni7HsWSdMWR2hASCCX3gEAYDJCEgCUEavVok6NaqpTo5p6dkgr/RGTkhuY/ohJUWa2vchtDUnRSekKj0q47HuoAABA2bCaXQAAVEYWi0Wtg3w0qX8LLZnUU8/f0LpE28WlpBc/CAAAlCtCEgA4QEiQb4nG+VdzK+dKAABAcQhJAOAAXYP9FOTroeLuNnpu0W59s/WELlzi0jwAAFC+CEkA4AA2q0VThoZIUoGglPO6uruLjiWc1+SFO9X/jVX6H2EJAABTEJIAwEEGhwZp5uiOCvT1yLc80NdDH4zuqE3P9dMzQ1rJr5qbjsSn6cmFOzXgzdVatP2Esu1FPy4BAACULWa3AwAHGhwapAEhgQqPSlBcSroCvD3UNdgvd9rvh3o11V1XN9LcDUf10epIRZ1J1eMLduqd3w7p0X7NNbR9XaYIBwCgnBGSAMDBbFbLJaf5rubuovG9m+qusEb6bP0RfbzmsA6fSdWkBTv09u8H9Vi/5rqhHWEJAIDywuV2AOCkqru7aEKfZlr7dF/9Y1BL1fBy1eHTqXrsqx0a+OYqfb/jJJfhAQBQDghJAODkcsLSmqf6aPLAFvL1dFXkn2Fp0FurtXjnKdkJSwAAlBlCEgBUEN4erprYt7nWPN1HTw5oIR8PFx2KO6dHvtyuQW+t1o+7CEsAAJQFQhIAVDA+Hq56pF9zrX2mrx7v30LeHi46GHdOE+dv1+D/rtZPu6IJSwAAXAFCEgBUUD4ernqsf3OtfbqvJvVvLm8PFx2IPacJ87dpyH/X6OfdhCUAAC4HIQkAKjhfT1dN6t9Ca5/uq8f6NZe3u4v2x6bo4S+26bq312jJHsISAAClQUgCgErC19NVjw+4GJYe/TMs/RGToofmbdP176zVkj0xhCUAAEqAkAQAlYyvl6ueGNBCa57uo0f6NlN1dxfti07WQ/O26oZ31mrp3hgZRv6wlG03tCEyXt/vOKkNkfFMLQ4AqNJ4mCwAVFI1vNz05MCWurd7sD5ZE6XZ66IUEZ2sBz/fqjZ1ffRYv+YaEFJHS/fGaNriCEUnpeduG+TroSlDQzQ4NMjEdwAAgDkISQBQydXwctPkQX+GpbWHNWfdEe09lawHPt+qBn6eOp5wvsA2MUnpGj9vm2aO7khQAgBUOVxuBwBVRM1qbvrHoFZa83Rfje/dVJ6u1kIDkiTlXGw3bXEEl94BAKocQhIAVDF+1dz09OBW+u+oDpccZ0iKTkpXeFSCYwoDAMBJEJIAoIo6n5VdonFxKenFDwIAoBIhJAFAFRXg7VGicb/sidbxhLRyrgYAAOdBSAKAKqprsJ+CfD1kKWbckj2x6vXvFZrwxTZtP3bWIbUBAGAmQhIAVFE2q0VThoZIUoGgZPnz65G+zdSjeS3ZDemn3dG68f31unnmei3ZE8OEDgCASospwAGgChscGqSZozsWeE5S4N+ek7QvOlmfrInSDztPasvRs9pydKsa+3vpnu7BurlTfXm58b8TAEDlwf/VAKCKGxwapAEhgQqPSlBcSroCvD3UNdhPNutf55daB/no9Vvb66nBLfXZ+iP6YtMxHYlP0wvf79Ubyw7ozm4NNSassQJ8SnafEwAAzoyQBACQzWpRWFP/YsfV8fHQU4NbaWLfZlq45YRmrY3SsYQ0vbciUh+vjtKwq+rq/h5N1DLQ2wFVAwBQPghJAIBS83Jz0ZhrGmv01Y20LCJGH6+J0tajZ/XN1hP6ZusJ9WheS/f3aKIezWvJYiluaggAAJwLIQkAcNlsVosGhwZpcGiQth07q0/WHNaSPTFac/CM1hw8o1aB3rqvRxMNa19Xbi7MFQQAqBj4PxYAoEx0bFhT79/ZSSsn99HYaxrLy82mP2JSNHnhTnWf8bveW3FIiWmZZpcJAECxCEkAgDLV0N9LU4e10YZn+unpwa1Ux8ddcSkZ+vfS/Qqb/rumfL9HR+NTzS4TAIAiEZIAAOXC18tV43s31Zqn+uqNW9urdZCPzmdl67MNR9X7Pyv10OdbtfVogtllAgBQAPckAQDKlZuLVTd1rK8bO9TT+sh4fbzmsFbuP60le2O0ZG+MOjSsoft7NNGgNoH5ph3PthvaFJWgrWcs8o9KUFizgHzrAQAoL4QkAIBDWCwWXduslq5tVksHYlM0a02UFm0/qe3HEvXwF9vUwM9T91wbrFs7N9Cag6fzPODWprkHtyjobw+4BQCgvHC5HQDA4VrU8daMm9tp3TN99WjfZqrp5arjCec1bXGEOr+8TA/N2/ZnQPpLTFK6xs/bpiV7ok2qGgBQVRCSAACmqe3tricGttT6Z/rp5RGhauzvpfNZ9kLHGn/+OW1xhLLtRqFjAAAoC4QkAIDpPN1sGn11I/3fjW0vOc6QFJ2UrvAoJnwAAJQfQhIAwGmcPpdRonFxKenFDwIA4DIRkgAATiPA26NE4/acTNKF7MIvywMA4EoRkgAATqNrsJ+CfD1U3ETfH6+J0nVvr9GK/XEyDO5PAgCULUISAMBp2KwWTRkaIkkFgpLlz69bOtVXDS9XHYg9p3GzN2v0rE3aeyrJ0aUCACoxQhIAwKkMDg3SzNEdFeib/9K7QF8PzRzdUf++pb1W/aOPHuzZRG42q9YditcN76zVk1/vVHTSeZOqBgBUJjxMFgDgdAaHBmlASKA2HIrTr2s2aWCPbgprFiCb9eL5JV9PVz17XWuNvrqR/r10v37YeUr/23ZCP+46pft7NNGDvZrI28PV5HcBAKioOJMEAHBKNqtF3YL91KmWoW7BfrkBKa8Gfl56+/YO+m7Ctera2E8ZF+x6d8Uh9f73Sn2+8SiTOwAALgshCQBQ4V3VoIYWPHi1Prqrk5rUqqb41Ew9/90eDXprtZZHxDK5AwCgVAhJAIBKwWKxaGCbQC19vKdeHN5GftXcFHk6VffN3aLbP96o3SeY3AEAUDKEJABApeJqs+rusMZa+Y/eerh3U7m7WLXxcIKGvrtWk77arhNn08wuEQDg5AhJAIBKycfDVU8NbqXfJ/fWTR3qSZK+23FKfV9fpVd/+UPJ6VkmVwgAcFaEJABApVavhqfeuO0q/fhId4U18VfmBbs+WBWpXq+t0Jx1UcpicgcAwN8QkgAAVUJoPV/Nv7+bPh3bWc0CqutsWpamLo7QwDdXa8meGCZ3AADkIiQBAKoMi8Wivq3qaMljPfTKjaGqVd1NUWdS9dC8rbr1ww3afuys2SUCAJwAIQkAUOW42Ky6s1sjrfxHHz3at5k8XK3afOSsbnx/vSbO36bjCUzuAABVGSEJAFBlVXd30RMDW2rl5D66pVN9WSzSj7ui1e/1VXrlpwglpeWf3CHbbmhDZLy+33FSGyLjlW3nEj0AqIxczC4AAACzBfp66N+3tNe4a4M1/Zd9WnPwjD5eE6Wvt5zQI32b6a6wRlrxR5ymLY5QdFJ67nZBvh6aMjREg0ODTKweAFDWOJMEAMCfQur66PN7u+mze7qqZR1vJZ3P0ss/7dO1r/6uh+ZtyxeQJCkmKV3j523Tkj3RJlUMACgPhCQAAP6mV4va+vmxHpoxsq1qV3fTmXOZhY7Ludhu2uIILr0DgEqEkAQAQCFsVotu69JQ/76l/SXHGZKik9IVHpXgmMIAAOWOkAQAwCUknc8qfpCkuJT04gcBACoEQhIAAJcQ4O1RonEr/ohTdNL5cq4GAOAIhCQAAC6ha7Cfgnw9ZClm3Hc7Tqn7jBWaOH+bth49K8PgHiUAqKgISQAAXILNatGUoSGSVCAoWf78erBnE3UL9lO23dCPu6I1cuZ6DX9vnRZtP6HMC3ZHlwwAuEKEJAAAijE4NEgzR3dUoG/+S+8CfT00c3RHPXtday14MEw/Pdpdt3auLzcXq3adSNLjC3bq2hm/663lB3Q6JcOk6gEApcXDZAEAKIHBoUEaEBKo8KgExaWkK8DbQ12D/WSz/nV+qU1dX712c3s9PbiVvgw/ps83HlVscobeWn5Q76+I1A3tgzTummC1re9r4jsBABSHkAQAQAnZrBaFNfUvdpx/dXdN7NtcD/Zqql/2xGj2uihtP5aob7ed1LfbTqpzo5oad22wBrWpIxcbF3UAgLMhJAEAUE5cbVYNa19Xw9rX1Y7jiZqzLko/7Y7WlqNnteXoWQX5euiusEa6vUtD1azmZna5AIA/8c9XAAA4wFUNauitUR207um+erRvM/lXc1N0UrpeW7JfYa/+pme/3aX9MSlmlwkAECEJAACHCvDx0BMDW2rdM331n1vaq01dH6Vn2fVl+HENemu17vh4o5ZFxCrbzhTiAGAWLrcDAMAEHq423dypvkZ2rKctR89q9rooLd0bq/WR8VofGa+Gfl66O6yRbu3SQD4ermaXCwBVCiEJAAATWSwWdWnspy6N/XQy8bw+33BUX4Yf07GENL380z69ueyAbu5UX2OuaawmtasX2D7bblxyxj0AQOkRkgAAcBL1anjqmSGt9Fi/5vpux0nNXhelA7Hn9NmGo/psw1H1bllbY69prJ7Na8tqtWjJnmhNWxyh6KT03H0E+XpoytAQDQ4NMvGdAEDFZuo9SatXr9bQoUNVt25dWSwWfffdd/nWG4ahF154QUFBQfL09FT//v118OBBc4oFAMBBPN1sur1rQy2d1FNf3NdN/VsHyGKRVu4/rbGzN6v/m6v0zP92afy8bfkCkiTFJKVr/LxtWrIn2qTqAaDiMzUkpaamqn379nrvvfcKXf/aa6/p7bff1gcffKBNmzapWrVqGjRokNLT0wsdDwBAZWKxWHRts1r6ZEwXrZzcW/dcGyxvdxcdPp2qrzYfV2FTO+Qsm7Y4gskfAOAymRqShgwZopdfflk33nhjgXWGYeitt97Sv/71Lw0fPlzt2rXT3LlzderUqQJnnAAAqOwa+VfTC0NDtOG5fhp7TeNLjjUkRSelKzwqwSG1AUBl47T3JEVFRSkmJkb9+/fPXebr66tu3bppw4YNGjVqVKHbZWRkKCMjI/d1cnKyJCkrK0tZWVnlW3QVl9Nf+uwY9Nvx6Lnj0fOC3K1Su3reJRobnZiqrCyfUu2fnjsW/XY8eu54ztTzktbgtCEpJiZGklSnTp18y+vUqZO7rjDTp0/XtGnTCiz/9ddf5eXlVbZFolDLli0zu4QqhX47Hj13PHqe3+EkiyRbseOWbtghy/HtupzJ7ui5Y9Fvx6PnjucMPU9LSyvROKcNSZfr2Wef1RNPPJH7Ojk5WQ0aNNDAgQPl41O6f01D6WRlZWnZsmUaMGCAXF15pkd5o9+OR88dj54XLttu6JvXVys2OaPQ+5JyLD1h0/7zXrq/e2MNv6qu3F2Kv8qenjsW/XY8eu54ztTznKvMiuO0ISkwMFCSFBsbq6Cgv6YxjY2N1VVXXVXkdu7u7nJ3dy+w3NXV1fQfSlVBrx2LfjsePXc8ep6fq6Spw9po/Lxtskj5glLOSaPr2gZp7aEzOhKfpn9+H6G3V0Tqvu5NdHu3hqruXvz//um5Y9Fvx6PnjucMPS/p9zd14oZLCQ4OVmBgoH777bfcZcnJydq0aZPCwsJMrAwAAPMNDg3SzNEdFejrkW95oK+HZo7uqPfu7Kj1z/TVv65vrUAfD8UmZ+iVn/fpmum/6fVf9yv+XEYRewYAmHom6dy5czp06FDu66ioKO3YsUN+fn5q2LChJk2apJdfflnNmzdXcHCwnn/+edWtW1cjRowwr2gAAJzE4NAgDQgJVHhUguJS0hXg7aGuwX6y/XkTUjV3F93Xo4nuDmus77af1AerInX4TKre+f2QPl5zWKO6NNR9PYJVvyb37AJAXqaGpC1btqhPnz65r3PuJRozZozmzJmjp556SqmpqXrggQeUmJio7t27a8mSJfLw8ChqlwAAVCk2q0VhTf0vOcbNxapbuzTQyE71tSwiRu+vjNSuE0mas/6I5m08qmFX1dVDvZqqRZ2SzZoHAJWdqSGpd+/eMoyibzm1WCx68cUX9eKLLzqwKgAAKieb1aLBoUEa1CZQ6yPj9f7KQ1p3KF7fbjupb7ed1ICQOrq/eyOzywQA0zntxA0AAKB8WCwWXduslq5tVks7jyfqg1WRWrI3RssiYrUsIlbNfKzybn5GfVoHymK5jPnDAaCCc9qJGwAAQPlr36CGZo7upGWP99KtnevL1WbRoWSr7pm7TTe8s1Y/7jqlbPulJhoHgMqHkAQAANQsoLpeu7m9fnu8h3oH2eXlZtPeU8maOH+7+r2+Ul+GH1PGhWyzywQAhyAkAQCAXEG+HrqxsV0rn+yhx/u3UE0vVx2JT9Oz3+5Wjxkr9OGqSKWkZ5ldJgCUK0ISAAAooKaXmx7r31zrnumrF24IUZCvh+JSMjT9lz907au/6z9L9+sMz1oCUEkxcQMAACiSl5uL7ukerNFXN9L3Oy4+aynydKreXXFIn6w9rNs6N9B9PZqogd9fz1rKthtFPrsJACoCQhIAACiWm4tVt3RuoJEd6+vXiFjNXHlIO08k6bMNRzVv0zENb19XD/Zqqqgz5zRtcYSik9Jztw3y9dCUoSEaHBpk4jsAgJIjJAEAgBKzWi0aHBqoQW3qaENkvGauitSag2f07faT+nb7yUK3iUlK1/h52zRzdEeCEoAKgXuSAABAqVksFl3TrJY+v7ebFk/sriGhdYocmzOB+LTFEUwnDqBCICQBAIAr0ra+r+4OC77kGENSdFK63l9xSNFJ52UYhCUAzovL7QAAwBWLS0kvfpCk15cd0OvLDsi/mptC6/mqbT3fi3/W91VdXw9ZLEzwAMB8hCQAAHDFArw9SjSuQU1PnUpKV3xqplYdOK1VB07nrvOr5qY2dX3UNk94ql/Tk+AEwOEISQAA4Ip1DfZTkK+HYpLSVdiFdBZJgb4eWvmPPsrKtuuPmBTtPpmkPSeStPtkkg7EpighNVNrDp7RmoNncrer4eWq0Lq+uWed2tbzVQO/0gcnpiUHUBqEJAAAcMVsVoumDA3R+HnbZJHyBaWcKDJlaIhsVotsVpuualBDVzWokTsmPStb+/8MTntPXQxO+2NSlJiWpbWHzmjtob+Ck6+nq0Lr+eQLT438vYoMTkv2RDvltOTZdkObohK09YxF/lEJCmsWQHADnAQhCQAAlInBoUGaObpjgUASWIJA4uFqU/sGNdQ+T3DKuJCtAzHntPvkxdC058/glHQ+S+sOxWvdofjcsd4eLn+GJp/c4NTYv5p+jYjR+HnbCpzdMnta8vzBzaa5B7c4RXADcBEhCQAAlJnBoUEaEBJYJpe2ubvY1Lb+xUkdcmResOtAbIr25AlO+2JSlJJ+QRsOx2vD4b+CU3U3mzKz7YVe/mfo4hmuaYsjNCAk0KFncJbsiXbK4AbgL4QkAABQpmxWi8Ka+pfLvt1crAr9c1KHUX8uy8r+KzjtOZms3SeTtC86Wecysy+5r5xpyXvM+F2+Xm5yd7HK3cUqNxer3F1sua/dXa1ys1nl7mq7+Gdhy/K8/msfOV9/LXOxWjX1hwinC24A8iMkAQCACs3VZlWbur5qU9dXt3W5uCwr265P1hzWjCX7i93+VFK6TiWVbArz8pYT3MKjEsotaAIoHiEJAABUOq42q65qULNEY5+/obWaBXgr84JdGReylZFlV2a2XRlZ2X/++efrC/mXZeRdl5WtjAv23H3k2y7r4jJ7KZ6f+9mGI3J3tapdPV+52KyX2QUAl4uQBAAAKqWSTks+9ppgh1zadiHbrjUHz2jcnM3Fjl2yJ0ZL9sTIx8NF3ZvXUo/mtdWzRW3Vq+FZ7nUCICQBAIBKqjTTkjuCi82qni1qXzK4SRenOL+mqZ/WHYpXcvoF/bw7Rj/vjpEkNa1dTT2a11avFrXVrYmfvNz4VQ4oD/yXBQAAKq0rmZa8PJQkuM0Y2VaDQ4N0IduuXSeTtPrAaa0+cFo7jicq8nSqIk+nas76I3KzWdW5cU31bFFbPZrXUkiQT6kfsgugcIQkAABQqZXltORlVU9JgpuLzaqODWuqY8OamtS/hZLOZ2n9oTNaffCMVh84rZOJ57U+Ml7rI+P16i9Sreru6tG8lnq2qKXuzWqrtre7Ke8PqAwISQAAoNIrz2nJL0dOcNtwKE6/rtmkgT26KaxZwCWDm6+nq4a0DdKQtkEyDENRZ1IvnmU6eEYbIuN15lyGFm0/qUXbT0qSQoJ81LNFbfVsXkudGteUu4ut2Lqy7YbThEnATIQkAAAAE9isFnUL9lP8PkPdShlGLBaLmtSuria1q2vstcHKuJCtrUfPavWBM1pz8LT2nkpWRPTFrw9WRcrT1aawpv5/nmmqrSa1qhW4NG/JnugCZ7eCTLosETAbIQkAAKCCc3ex6ZqmtXRN01p6ZkgrnU7J0LpDZ3LPNJ05l6Hf/4jT73/ESZLq1fBUzxa11LN5bV3TtJY2HD6j8fO2FZhMIiYpXePnbdPM0R0JSqhSCEkAAACVTG1vd43oUE8jOtSTYRjaF52i1QdPa83B09ocdVYnE8/ry/Dj+jL8uCySXGyWQmfbM3RxQolpiyM0ICSQS+9QZRCSAAAAKjGLxaKQuj4Kqeujh3o1VVrmBW2KSsidNS/ydKqysot+0q0hKTopXeFRCU51XxdQnghJAAAAVYiXm4v6tAxQn5YBkqQ566M09YeIYreLS0kvdgxQWVjNLgAAAADmaVnHp0Tj0rOyy7kSwHkQkgAAAKqwrsF+CvL1UHF3Gz39v92aMH+bDsamOKQuwEyEJAAAgCrMZrVoytAQSSoQlHJed2xUQ5L0065oDXxrtSZ9tV1RZ1IdViPgaIQkAACAKm5waJBmju6oQF+PfMsDfT30weiO+nb8tVoyqYcGtakjw5C+23FK/d9YpckLd+p4QppJVQPlh4kbAAAAoMGhQRoQEqjwqATFpaQrwNtDXfM85LZVoI8+vKuz9pxM0pvLDui3P+L0zdYT+m77Sd3SuYEm9m2mejU8TX4XQNkgJAEAAEDSxUvvipvmO7Ser2aN7aLtx87qzeUHtfrAaX0Zfkz/23pCo7o20MO9mxU4IwVUNFxuBwAAgFLr0LCm5t7TVQsfClNYE39lZts1d8NR9fz3Cr24OEKnUzLMLhG4bIQkAAAAXLYujf305QNXa/793dS5UU1lXrDr03VR6vnaCk3/ZZ8SUjPNLhEoNUISAAAArtg1TWtp4UNhmntPV13VoIbOZ2Xrw1WH1WPG7/rP0v1KTCMsoeIgJAEAAKBMWCwW9WxRW4sevkafju2s0Ho+Ss3M1rsrDqnHjBV6a/kBJadnmV0mUCxCEgAAAMqUxWJR31Z1tHhid314Vye1CvRWSsYFvbX8oHrMWKH3VhxSasYFs8sEikRIAgAAQLmwWCwa1CZQPz/aQ+/d0VHNAqor6XyW/r10v3q8tkIfrorU+cxss8sECiAkAQAAoFxZrRZd3y5ISyf11Fu3XaXgWtWUkJqp6b/8oR6vrdCna6OUnlWxwlK23dCGyHh9v+OkNkTGK9tumF0SyhDPSQIAAIBD2KwWjehQTze0C9Ki7Sf19u8HdTzhvF78MUIfro7UxD7NdGuXBnJ3seVuk203inzArVmW7InWtMURik5Kz10W5OuhKUNDNDg0yLS6nLFXFRUhCQAAAA7lYrPqls4NNKJDPS3cckLv/n5Qp5LS9fz3e/XBqsOa2LeZbu5UX7/ti3W6MLJkT7TGz9umv583iklK1/h52zRzdEdTanPW4FZRcbkdAAAATOFqs+qObg214h+99eLwNgrwdtfJxPN69tvdCpv+mx6aty3fL/3SX2FkyZ5oh9ebbTc0bXFEgYAkKXfZtMURDr/0Lie4OVOvKjrOJAEAAMBU7i423R3WWLd2bqAvNh3T+ysO6sy5wp+rZEiy6GIYGRASmHs5mWEYyso2lHEhWxkX7Be/srKVmW1XRtafry9kKzU9UzviLcracUoXDIsyLtiV+ee6S22XccGu0ykZBYLI32uLTkpX3/+slI+nq6xWi2yWi5cZ5nxZLX/+3WL5c/2fy/8cW3BZ3u1UYJnVIn28JqrI4FZYr1A8QhIAAACcgoerTfd2D1bT2tU0dvbmIsflhJGOLy2TYRi54abkbNKBPVdcb1GOJqSV275LK6dX7/5+UDd1rK/6NT1lsRCWikNIAgAAgFNJOl+yB85eapybi1XuLla5u9j+/NMqNxer3FwsSk1KUlCdWvJwtf213vXi2L+2+3NbV6vcbBfXH41P01vLDxZb1zNDWqllHW9l2w1lG4bsdkMX7IbshnFxWe7flbs+O+/63GV51ufZT96xh8+c04bIhGJrenP5Qb25/KB8PFwUUtdHber6qs2ffzatXU0uNu7CyYuQBAAAAKcS4O1RonEzRrZV58Z++QKNu8vFUFPU2ZKsrCz9/PPPuu66TnJ1dS1VXdl2Qws2H1dMUnqhl7dZJAX6euj+Hk0cdmnbhsh4bYjcWOy4xrW8dOpsupLTL2jj4QRtPPxXsHJ3sapVoLdC6voqtN7F4NQq0FserrZL7LFyIyQBAADAqXQN9lOQr0exYeTmTg0cep+NzWrRlKEhGj9vmyxSvtpyqpgyNMShNZW0V7890VvZdkMH41K091SyIk4la++pJEWcSlZqZrZ2nkjSzhNJudvZrBY1rV0t94xTSF0ftQnyla9X6YPlpqgEbT1jkX9UgsKaBVSIe6MISQAAAHAqzhhGcgwODdLM0R0LTLcdaNJ026Xplc1q+TP0+OaOsdsNHU1I095TSdp7Kvni18kkxadm6kDsOR2IPadF20/mjq9f01Nt6vootK6v2vx51inA273QM3f5pyW3ae7BLRVmWnJCEgAAAJyOs4WRv9c2ICTQaR7ceiW9slotCq5VTcG1qumGdnUlXZwpMDY5I09wuvjnibPnc7+W7o3N3Uet6m4Kyb3H6WJw2ncqWRPmO9/zpEqKkAQAAACn5GxhJC+b1aKwpv5ml5GrLHtlsVgU6OuhQF8P9WtdJ3d5UlqW9kZfvERvz8mLwSny9DmdOZep1QdOa/WB03/tQ6rQ05ITkgAAAOC0nC2MOLPy7pWvl6uuaVpL1zStlbvsfGa2/ohJzr1UL+LPs04XLvFA3ZxpycOjEpz2Z0tIAgAAAHBZPN1s6tCwpjo0rJm7bNG2E3r8653FbhuXUvSDec3GhOgAAAAAykygr2eJxpV0qnczEJIAAAAAlJmcacmLutvIIinI9+I9U86KkAQAAACgzORMSy6pQFAyewr3kiIkAQAAAChTOdOSB/rmv6Qu0NfD6af/lpi4AQAAAEA5yJmWfMOhOP26ZpMG9uimsGYBTn0GKQchCQAAAEC5sFkt6hbsp/h9hro5yTOuSoLL7QAAAAAgD0ISAAAAAORBSAIAAACAPAhJAAAAAJAHIQkAAAAA8iAkAQAAAEAehCQAAAAAyIOQBAAAAAB5EJIAAAAAIA9CEgAAAADk4dQhaerUqbJYLPm+WrVqZXZZAAAAACoxF7MLKE6bNm20fPny3NcuLk5fMgAAAIAKzOkTh4uLiwIDA80uAwAAAEAV4fQh6eDBg6pbt648PDwUFham6dOnq2HDhkWOz8jIUEZGRu7rpKQkSVJCQoKysrLKvd6qLCsrS2lpaYqPj5erq6vZ5VR69Nvx6Lnj0XPHo+eORb8dj547njP1PCUlRZJkGMYlx1mM4kaY6JdfftG5c+fUsmVLRUdHa9q0aTp58qT27Nkjb2/vQreZOnWqpk2b5uBKAQAAAFQUx48fV/369Ytc79Qh6e8SExPVqFEjvfHGG7r33nsLHfP3M0l2u10JCQny9/eXxWJxVKlVUnJysho0aKDjx4/Lx8fH7HIqPfrtePTc8ei549Fzx6LfjkfPHc+Zem4YhlJSUlS3bl1ZrUXPYef0l9vlVaNGDbVo0UKHDh0qcoy7u7vc3d0LbAfH8fHxMf0/gKqEfjsePXc8eu549Nyx6Lfj0XPHc5ae+/r6FjvGqacA/7tz584pMjJSQUFBZpcCAAAAoJJy6pA0efJkrVq1SkeOHNH69et14403ymaz6fbbbze7NAAAAACVlFNfbnfixAndfvvtio+PV+3atdW9e3dt3LhRtWvXNrs0FMLd3V1TpkwpcLkjygf9djx67nj03PHouWPRb8ej545XEXteoSZuAAAAAIDy5tSX2wEAAACAoxGSAAAAACAPQhIAAAAA5EFIAgAAAIA8CEkokenTp6tLly7y9vZWQECARowYof37919ymzlz5shiseT78vDwcFDFFd/UqVML9K9Vq1aX3GbhwoVq1aqVPDw81LZtW/38888Oqrbia9y4cYF+WywWTZgwodDxHN+lt3r1ag0dOlR169aVxWLRd999l2+9YRh64YUXFBQUJE9PT/Xv318HDx4sdr/vvfeeGjduLA8PD3Xr1k3h4eHl9A4qnkv1PCsrS08//bTatm2ratWqqW7durr77rt16tSpS+7zcj6bqorijvGxY8cW6N3gwYOL3S/HeNGK63lhn+sWi0X//ve/i9wnx3jRSvL7YHp6uiZMmCB/f39Vr15dI0eOVGxs7CX3e7mf/+WJkIQSWbVqlSZMmKCNGzdq2bJlysrK0sCBA5WamnrJ7Xx8fBQdHZ37dfToUQdVXDm0adMmX//Wrl1b5Nj169fr9ttv17333qvt27drxIgRGjFihPbs2ePAiiuuzZs35+v1smXLJEm33HJLkdtwfJdOamqq2rdvr/fee6/Q9a+99prefvttffDBB9q0aZOqVaumQYMGKT09vch9LliwQE888YSmTJmibdu2qX379ho0aJDi4uLK621UKJfqeVpamrZt26bnn39e27Zt07fffqv9+/dr2LBhxe63NJ9NVUlxx7gkDR48OF/vvvzyy0vuk2P80orred5eR0dH69NPP5XFYtHIkSMvuV+O8cKV5PfBxx9/XIsXL9bChQu1atUqnTp1SjfddNMl93s5n//lzgAuQ1xcnCHJWLVqVZFjZs+ebfj6+jquqEpmypQpRvv27Us8/tZbbzWuv/76fMu6detmPPjgg2VcWdXw2GOPGU2bNjXsdnuh6zm+r4wkY9GiRbmv7Xa7ERgYaPz73//OXZaYmGi4u7sbX375ZZH76dq1qzFhwoTc19nZ2UbdunWN6dOnl0vdFdnfe16Y8PBwQ5Jx9OjRIseU9rOpqiqs32PGjDGGDx9eqv1wjJdcSY7x4cOHG3379r3kGI7xkvv774OJiYmGq6ursXDhwtwx+/btMyQZGzZsKHQfl/v5X944k4TLkpSUJEny8/O75Lhz586pUaNGatCggYYPH669e/c6orxK4+DBg6pbt66aNGmiO++8U8eOHSty7IYNG9S/f/98ywYNGqQNGzaUd5mVTmZmpubNm6d77rlHFoulyHEc32UnKipKMTEx+Y5hX19fdevWrchjODMzU1u3bs23jdVqVf/+/TnuL1NSUpIsFotq1KhxyXGl+WxCfitXrlRAQIBatmyp8ePHKz4+vsixHONlKzY2Vj/99JPuvffeYsdyjJfM338f3Lp1q7KysvIds61atVLDhg2LPGYv5/PfEQhJKDW73a5Jkybp2muvVWhoaJHjWrZsqU8//VTff/+95s2bJ7vdrmuuuUYnTpxwYLUVV7du3TRnzhwtWbJEM2fOVFRUlHr06KGUlJRCx8fExKhOnTr5ltWpU0cxMTGOKLdS+e6775SYmKixY8cWOYbju2zlHKelOYbPnDmj7Oxsjvsykp6erqefflq33367fHx8ihxX2s8m/GXw4MGaO3eufvvtN82YMUOrVq3SkCFDlJ2dXeh4jvGy9dlnn8nb27vYS784xkumsN8HY2Ji5ObmVuAfWi51zF7O578juJj2nVFhTZgwQXv27Cn2+tywsDCFhYXlvr7mmmvUunVrffjhh3rppZfKu8wKb8iQIbl/b9eunbp166ZGjRrp66+/LtG/guHyzZo1S0OGDFHdunWLHMPxjcokKytLt956qwzD0MyZMy85ls+myzdq1Kjcv7dt21bt2rVT06ZNtXLlSvXr18/EyqqGTz/9VHfeeWexk+xwjJdMSX8frKg4k4RSmThxon788UetWLFC9evXL9W2rq6u6tChgw4dOlRO1VVuNWrUUIsWLYrsX2BgYIHZY2JjYxUYGOiI8iqNo0ePavny5brvvvtKtR3H95XJOU5LcwzXqlVLNpuN4/4K5QSko0ePatmyZZc8i1SY4j6bULQmTZqoVq1aRfaOY7zsrFmzRvv37y/1Z7vEMV6Yon4fDAwMVGZmphITE/ONv9Qxezmf/45ASEKJGIahiRMnatGiRfr9998VHBxc6n1kZ2dr9+7dCgoKKocKK79z584pMjKyyP6FhYXpt99+y7ds2bJl+c52oHizZ89WQECArr/++lJtx/F9ZYKDgxUYGJjvGE5OTtamTZuKPIbd3NzUqVOnfNvY7Xb99ttvHPcllBOQDh48qOXLl8vf37/U+yjuswlFO3HihOLj44vsHcd42Zk1a5Y6deqk9u3bl3pbjvG/FPf7YKdOneTq6prvmN2/f7+OHTtW5DF7OZ//DmHalBGoUMaPH2/4+voaK1euNKKjo3O/0tLScsfcddddxjPPPJP7etq0acbSpUuNyMhIY+vWrcaoUaMMDw8PY+/evWa8hQrnySefNFauXGlERUUZ69atM/r372/UqlXLiIuLMwyjYL/XrVtnuLi4GP/5z3+Mffv2GVOmTDFcXV2N3bt3m/UWKpzs7GyjYcOGxtNPP11gHcf3lUtJSTG2b99ubN++3ZBkvPHGG8b27dtzZ1J79dVXjRo1ahjff/+9sWvXLmP48OFGcHCwcf78+dx99O3b13jnnXdyX3/11VeGu7u7MWfOHCMiIsJ44IEHjBo1ahgxMTEOf3/O6FI9z8zMNIYNG2bUr1/f2LFjR77P9oyMjNx9/L3nxX02VWWX6ndKSooxefJkY8OGDUZUVJSxfPlyo2PHjkbz5s2N9PT03H1wjJdOcZ8rhmEYSUlJhpeXlzFz5sxC98ExXnIl+X3woYceMho2bGj8/vvvxpYtW4ywsDAjLCws335atmxpfPvtt7mvS/L572iEJJSIpEK/Zs+enTumV69expgxY3JfT5o0yWjYsKHh5uZm1KlTx7juuuuMbdu2Ob74Cuq2224zgoKCDDc3N6NevXrGbbfdZhw6dCh3/d/7bRiG8fXXXxstWrQw3NzcjDZt2hg//fSTg6uu2JYuXWpIMvbv319gHcf3lVuxYkWhnyM5fbXb7cbzzz9v1KlTx3B3dzf69etX4GfRqFEjY8qUKfmWvfPOO7k/i65duxobN2500DtyfpfqeVRUVJGf7StWrMjdx997XtxnU1V2qX6npaUZAwcONGrXrm24uroajRo1Mu6///4CYYdjvHSK+1wxDMP48MMPDU9PTyMxMbHQfXCMl1xJfh88f/688fDDDxs1a9Y0vLy8jBtvvNGIjo4usJ+825Tk89/RLIZhGOVzjgoAAAAAKh7uSQIAAACAPAhJAAAAAJAHIQkAAAAA8iAkAQAAAEAehCQAAAAAyIOQBAAAAAB5EJIAAAAAIA9CEgAAAADkQUgCAOASLBaLvvvuO7PLAAA4ECEJAOC0xo4dK4vFUuBr8ODBZpcGAKjEXMwuAACASxk8eLBmz56db5m7u7tJ1QAAqgLOJAEAnJq7u7sCAwPzfdWsWVPSxUvhZs6cqSFDhsjT01NNmjTRN998k2/73bt3q2/fvvL09JS/v78eeOABnTt3Lt+YTz/9VG3atJG7u7uCgoI0ceLEfOvPnDmjG2+8UV5eXmrevLl++OGH8n3TAABTEZIAABXa888/r5EjR2rnzp268847NWrUKO3bt0+SlJqaqkGDBqlmzZravHmzFi5cqOXLl+cLQTNnztSECRP0wAMPaPfu3frhhx/UrFmzfN9j2rRpuvXWW7Vr1y5dd911uvPOO5WQkODQ9wkAcByLYRiG2UUAAFCYsWPHat68efLw8Mi3/LnnntNzzz0ni8Wihx56SDNnzsxdd/XVV6tjx456//339fHHH+vpp5/W8ePHVa1aNUnSzz//rKFDh+rUqVOqU6eO6tWrp3Hjxunll18utAaLxaJ//etfeumllyRdDF7Vq1fXL7/8wr1RAFBJcU8SAMCp9enTJ18IkiQ/P7/cv4eFheVbFxYWph07dkiS9u3bp/bt2+cGJEm69tprZbfbtX//flksFp06dUr9+vW7ZA3t2rXL/Xu1atXk4+OjuLi4y31LAAAnR0gCADi1atWqFbj8rax4enqWaJyrq2u+1xaLRXa7vTxKAgA4Ae5JAgBUaBs3bizwunXr1pKk1q1ba+fOnUpNTc1dv27dOlmtVrVs2VLe3t5q3LixfvvtN4fWDABwbpxJAgA4tYyMDMXExORb5uLiolq1akmSFi5cqM6dO6t79+764osvFB4erlmzZkmS7rzzTk2ZMkVjxozR1KlTdfr0aT3yyCO66667VKdOHUnS1KlT9dBDDykgIEBDhgxRSkqK1q1bp0ceecSxbxQA4DQISQAAp7ZkyRIFBQXlW9ayZUv98ccfki7OPPfVV1/p4YcfVlBQkL788kuFhIRIkry8vLR06VI99thj6tKli7y8vDRy5Ei98cYbufsaM2aM0tPT9eabb2ry5MmqVauWbr75Zse9QQCA02F2OwBAhWWxWLRo0SKNGDHC7FIAAJUI9yQBAAAAQB6EJAAAAADIg3uSAAAVFleMAwDKA2eSAAAAACAPQhIAAAAA5EFIAgAAAIA8CEkAAAAAkAchCQAAAADyICQBAAAAQB6EJAAAAADIg5AEAAAAAHn8P5wHG6yFc+VLAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Avoid parallelism error from HuggingFace during training\n",
    "tokenizer.parallelism = False\n",
    "\n",
    "# Train the model using FHE simulation\n",
    "train_custom_model(hybrid_model, train_dataloader, training_args, tokenizer, fhe=\"disable\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "65d448c8",
   "metadata": {},
   "source": [
    "Note that our goal is to showcase the use of FHE for encrypted fine-tuning. The dataset consists of 68 examples and a total of 2,386 tokens, which is relatively small. Despite its limited size, which offers little support for the model's learning process, it still manages to produce interesting results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "bd666f38",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the fine-tuned model\n",
    "fine_tuned_model = hybrid_model.model.inference_model\n",
    "\n",
    "# Set FHE mode to disable for text generation\n",
    "hybrid_model.set_fhe_mode(\"execute\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3e91ad0b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prompt: who invented FHE?\n",
      "Response: Fully Homomorphic Encryption (HE) allows users to perform complex computations on encrypted data without having to decode them. This allows for\n",
      "\n"
     ]
    },
    {
     "data": {},
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Inference using the fine-tuned model with LoRA weights\n",
    "# Seed for reproducibility\n",
    "torch.manual_seed(SEED)\n",
    "\n",
    "fine_tuned_model.enable_adapter_layers()\n",
    "prompt = \"who invented FHE?\"\n",
    "_ = generate_and_print(prompt, fine_tuned_model, tokenizer, SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "21e2a1d1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prompt: What is FHE?\n",
      "Response: The first, the first,\n",
      "\n",
      "\n",
      "A.\n",
      "\n",
      "The first,\n",
      "\n",
      ".\n",
      "\n",
      ".\n",
      "\n",
      ".\n",
      "The\n",
      "\n"
     ]
    },
    {
     "data": {},
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Original inference without LoRA weights\n",
    "# Seed for reproducibility\n",
    "torch.manual_seed(SEED)\n",
    "\n",
    "peft_model.disable_adapter_layers()\n",
    "\n",
    "prompt = \"What is FHE?\"\n",
    "generate_and_print(prompt, peft_model, tokenizer, SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "c97425ee",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of weights: 124734720\n",
      "Total number of LoRA weights: 294912\n"
     ]
    }
   ],
   "source": [
    "peft_model.enable_adapter_layers()\n",
    "\n",
    "# Print weights and model size\n",
    "total_weights_size = print_weights_and_size(hybrid_model.model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "31367ff5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the model\n",
    "path = Path(\"deployment/gpt2_lora_finetuned\")\n",
    "path.mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "if path.is_dir() and any(path.iterdir()):\n",
    "    shutil.rmtree(path)\n",
    "\n",
    "hybrid_model.save_and_clear_private_info(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "a1dda636",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of weights: 39717120\n",
      "Total number of LoRA weights: 294912\n"
     ]
    }
   ],
   "source": [
    "# Print weights and size after saving\n",
    "total_weights_size_private = print_weights_and_size(hybrid_model.model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "506ad2f5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total weights removed: 68.16 %\n"
     ]
    }
   ],
   "source": [
    "# Calculate and print the percentage of weights removed\n",
    "percentage_removed = (total_weights_size - total_weights_size_private) / total_weights_size * 100\n",
    "print(f\"Total weights removed: {percentage_removed:.2f} %\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "465cb18b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Note: Around 95% of the remaining weights are from the embedding layers (wpe and wte)\n",
    "# as well as the final lm_head layer."
   ]
  }
 ],
 "metadata": {
  "execution": {
   "timeout": 10800
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
